Merge pull request #298 from bgrozev/conn-quality

Conn quality

JitsiConference.js

@@ -21,6 +21,7 @@ var Transcriber = require("./modules/transcription/transcriber");
 import ParticipantConnectionStatus
     from "./modules/connectivity/ParticipantConnectionStatus";
 import TalkMutedDetection from "./modules/TalkMutedDetection";
+import ConnectionQuality from "./modules/connectivity/ConnectionQuality";
 
 /**
  * Creates a JitsiConference object with the given name and properties.
@@ -64,6 +65,20 @@ function JitsiConference(options) {
     // We need to know if the potential issue happened before or after
     // the restart.
     this.wasStopped = false;
+
+    /**
+     * The object which monitors local and remote connection statistics (e.g.
+     * sending bitrate) and calculates a number which represents the connection
+     * quality.
+     */
+    this.connectionQuality
+        = new ConnectionQuality(this, this.eventEmitter, options);
+
+    /**
+     * Indicates whether the connection is interrupted or not.
+     */
+    this.connectionIsInterrupted = false;
+
 }
 
 /**
@@ -229,17 +244,24 @@ JitsiConference.prototype.getExternalAuthUrl = function (urlForPopup) {
 };
 
 /**
- * Returns the local tracks.
+ * Returns the local tracks of the given media type, or all local tracks if no
+ * specific type is given.
+ * @param mediaType {MediaType} Optional media type (audio or video).
  */
-JitsiConference.prototype.getLocalTracks = function () {
+JitsiConference.prototype.getLocalTracks = function (mediaType) {
+    let tracks = [];
     if (this.rtc) {
-        return this.rtc.localTracks.slice();
-    } else {
-        return [];
+        tracks = this.rtc.localTracks.slice();
     }
+    if (mediaType !== undefined) {
+        tracks = tracks.filter(
+            (track) => {
+                return track && track.getType && track.getType() === mediaType;
+            });
+    }
+    return tracks;
 };
 
-
 /**
  * Attaches a handler for events(For example - "participant joined".) in the conference. All possible event are defined
  * in JitsiConferenceEvents.
@@ -1247,4 +1269,8 @@ JitsiConference.prototype.broadcastEndpointMessage = function (payload) {
     this.sendEndpointMessage("", payload);
 };
 
+JitsiConference.prototype.isConnectionInterrupted = function () {
+    return this.connectionIsInterrupted;
+};
+
 module.exports = JitsiConference;

JitsiConferenceEventManager.js

@@ -99,19 +99,22 @@ JitsiConferenceEventManager.prototype.setupChatRoomListeners = function () {
         JitsiConferenceEvents.CONFERENCE_JOINED);
     // send some analytics events
     chatRoom.addListener(XMPPEvents.MUC_JOINED,
-        function ()
-        {
-            for (var ckey in chatRoom.connectionTimes){
-                var cvalue = chatRoom.connectionTimes[ckey];
-                Statistics.analytics.sendEvent('conference.' + ckey,
-                    {value: cvalue});
+        () => {
+            let key, value;
+
+            this.conference.connectionIsInterrupted = false;
+
+            for (key in chatRoom.connectionTimes){
+                value = chatRoom.connectionTimes[key];
+                Statistics.analytics.sendEvent('conference.' + key,
+                    {value: value});
             }
-            for (var xkey in chatRoom.xmpp.connectionTimes){
-                var xvalue = chatRoom.xmpp.connectionTimes[xkey];
-                Statistics.analytics.sendEvent('xmpp.' + xkey,
-                    {value: xvalue});
+            for (key in chatRoom.xmpp.connectionTimes){
+                value = chatRoom.xmpp.connectionTimes[key];
+                Statistics.analytics.sendEvent('xmpp.' + key,
+                    {value: value});
             }
-        });
+    });
 
     this.chatRoomForwarder.forward(XMPPEvents.ROOM_JOIN_ERROR,
         JitsiConferenceEvents.CONFERENCE_FAILED,
@@ -195,8 +198,9 @@ JitsiConferenceEventManager.prototype.setupChatRoomListeners = function () {
     this.chatRoomForwarder.forward(XMPPEvents.CONNECTION_INTERRUPTED,
         JitsiConferenceEvents.CONNECTION_INTERRUPTED);
     chatRoom.addListener(XMPPEvents.CONNECTION_INTERRUPTED,
-        function () {
+        () => {
             Statistics.sendEventToAll('connection.interrupted');
+            this.conference.connectionIsInterrupted = true;
         });
 
     this.chatRoomForwarder.forward(XMPPEvents.RECORDER_STATE_CHANGED,
@@ -208,8 +212,9 @@ JitsiConferenceEventManager.prototype.setupChatRoomListeners = function () {
     this.chatRoomForwarder.forward(XMPPEvents.CONNECTION_RESTORED,
         JitsiConferenceEvents.CONNECTION_RESTORED);
     chatRoom.addListener(XMPPEvents.CONNECTION_RESTORED,
-        function () {
+        () => {
             Statistics.sendEventToAll('connection.restored');
+            this.conference.connectionIsInterrupted = false;
         });
 
     this.chatRoomForwarder.forward(XMPPEvents.CONFERENCE_SETUP_FAILED,

JitsiConferenceEvents.js

@@ -42,6 +42,7 @@ export const CONNECTION_INTERRUPTED = "conference.connectionInterrupted";
 export const CONNECTION_RESTORED = "conference.connectionRestored";
 /**
  * New local connection statistics are received.
+ * @deprecated Use ConnectionQualityEvents.LOCAL_STATS_UPDATED instead.
  */
 export const CONNECTION_STATS = "conference.connectionStats";
 /**

JitsiMeetJS.js

@@ -7,6 +7,7 @@ import * as JitsiConferenceEvents from "./JitsiConferenceEvents";
 import * as JitsiConnectionErrors from "./JitsiConnectionErrors";
 import * as JitsiConnectionEvents from "./JitsiConnectionEvents";
 import * as JitsiMediaDevicesEvents from "./JitsiMediaDevicesEvents";
+import * as ConnectionQualityEvents from "./service/connectivity/ConnectionQualityEvents";
 import JitsiTrackError from "./JitsiTrackError";
 import * as JitsiTrackErrors from "./JitsiTrackErrors";
 import * as JitsiTrackEvents from "./JitsiTrackEvents";
@@ -76,7 +77,8 @@ var LibJitsiMeet = {
         conference: JitsiConferenceEvents,
         connection: JitsiConnectionEvents,
         track: JitsiTrackEvents,
-        mediaDevices: JitsiMediaDevicesEvents
+        mediaDevices: JitsiMediaDevicesEvents,
+        connectionQuality: ConnectionQualityEvents
     },
     errors: {
         conference: JitsiConferenceErrors,

modules/RTC/RTCBrowserType.js

@@ -147,6 +147,14 @@ var RTCBrowserType = {
      */
     isAndroid: function() {
         return isAndroid;
+    },
+
+    /**
+     * Whether jitsi-meet supports simulcast on the current browser.
+     * @returns {boolean}
+     */
+    supportsSimulcast: function() {
+        return RTCBrowserType.isChrome();
     }
 
     // Add version getters for other browsers when needed

modules/connectivity/ConnectionQuality.js

@@ -0,0 +1,435 @@
+import * as ConnectionQualityEvents
+    from "../../service/connectivity/ConnectionQualityEvents";
+import * as ConferenceEvents from "../../JitsiConferenceEvents";
+import {getLogger} from "jitsi-meet-logger";
+import RTCBrowserType from "../RTC/RTCBrowserType";
+
+var XMPPEvents = require('../../service/xmpp/XMPPEvents');
+var MediaType = require('../../service/RTC/MediaType');
+var VideoType = require('../../service/RTC/VideoType');
+var Resolutions = require("../../service/RTC/Resolutions");
+
+const logger = getLogger(__filename);
+
+/**
+ * The value to use for the "type" field for messages sent by ConnectionQuality
+ * over the data channel.
+ */
+const STATS_MESSAGE_TYPE = "stats";
+
+/**
+ * See media/engine/simulcast.ss from webrtc.org
+ */
+const kSimulcastFormats = [
+    { width: 1920, height: 1080, layers:3, max: 5000, target: 4000, min: 800 },
+    { width: 1280, height: 720,  layers:3, max: 2500, target: 2500, min: 600 },
+    { width: 960,  height: 540,  layers:3, max: 900,  target: 900, min: 450 },
+    { width: 640,  height: 360,  layers:2, max: 700,  target: 500, min: 150 },
+    { width: 480,  height: 270,  layers:2, max: 450,  target: 350, min: 150 },
+    { width: 320,  height: 180,  layers:1, max: 200,  target: 150, min: 30 }
+];
+
+/**
+ * The initial bitrate for video in kbps.
+ */
+var startBitrate = 800;
+
+/**
+ * Gets the expected bitrate (in kbps) in perfect network conditions.
+ * @param simulcast {boolean} whether simulcast is enabled or not.
+ * @param resolution {Resolution} the resolution.
+ * @param millisSinceStart {number} the number of milliseconds since sending
+ * video started.
+ */
+function getTarget(simulcast, resolution, millisSinceStart) {
+    // Completely ignore the bitrate in the first 5 seconds, as the first
+    // event seems to fire very early and the value is suspicious and causes
+    // false positives.
+    if (millisSinceStart < 5000) {
+        return 1;
+    }
+
+    let target = 0;
+    let height = Math.min(resolution.height, resolution.width);
+
+    if (simulcast) {
+        // Find the first format with height no bigger than ours.
+        let simulcastFormat = kSimulcastFormats.find(f => f.height <= height);
+        if (simulcastFormat) {
+            // Sum the target fields from all simulcast layers for the given
+            // resolution (e.g. 720p + 360p + 180p).
+            for (height = simulcastFormat.height; height >= 180; height /=2) {
+                simulcastFormat
+                    = kSimulcastFormats.find(f => f.height == height);
+                if (simulcastFormat) {
+                    target += simulcastFormat.target;
+                } else {
+                    break;
+                }
+            }
+        }
+    } else {
+        // See GetMaxDefaultVideoBitrateKbps in
+        // media/engine/webrtcvideoengine2.cc from webrtc.org
+        let pixels = resolution.width * resolution.height;
+        if (pixels <= 320 * 240) {
+            target = 600;
+        } else if (pixels <= 640 * 480) {
+            target =  1700;
+        } else if (pixels <= 960 * 540) {
+            target = 2000;
+        } else {
+            target = 2500;
+        }
+    }
+
+    // Allow for an additional 1 second for ramp up -- delay any initial drop
+    // of connection quality by 1 second.
+    return Math.min(target, rampUp(Math.max(0, millisSinceStart - 1000)));
+}
+
+/**
+ * Gets the bitrate to which GCC would have ramped up in perfect network
+ * conditions after millisSinceStart milliseconds.
+ * @param millisSinceStart {number} the number of milliseconds since sending
+ * video was enabled.
+ */
+function rampUp(millisSinceStart) {
+    if (millisSinceStart > 60000) {
+        return Number.MAX_SAFE_INTEGER;
+    }
+    // According to GCC the send side bandwidth estimation grows with at most
+    // 8% per second.
+    // https://tools.ietf.org/html/draft-ietf-rmcat-gcc-02#section-5.5
+    return startBitrate * Math.pow(1.08, millisSinceStart / 1000);
+}
+
+/**
+ * A class which monitors the local statistics coming from the RTC modules, and
+ * calculates a "connection quality" value, in percent, for the media
+ * connection. A value of 100% indicates a very good network connection, and a
+ * value of 0% indicates a poor connection.
+ */
+export default class ConnectionQuality {
+    constructor(conference, eventEmitter, options) {
+        this.eventEmitter = eventEmitter;
+
+        /**
+         * The owning JitsiConference.
+         */
+        this._conference = conference;
+
+        /**
+         * Whether simulcast is supported. Note that even if supported, it is
+         * currently not used for screensharing, which is why we have an
+         * additional check.
+         */
+        this._simulcast
+            = !options.disableSimulcast && RTCBrowserType.supportsSimulcast();
+
+        /**
+         * Holds statistics about the local connection quality.
+         */
+        this._localStats = {connectionQuality: 100};
+
+        /**
+         * The time this._localStats.connectionQuality was last updated.
+         */
+        this._lastConnectionQualityUpdate = -1;
+
+        /**
+         * Maps a participant ID to an object holding connection quality
+         * statistics received from this participant.
+         */
+        this._remoteStats = {};
+
+        /**
+         * The time that the ICE state last changed to CONNECTED. We use this
+         * to calculate how much time we as a sender have had to ramp-up.
+         */
+        this._timeIceConnected = -1;
+
+        /**
+         * The time that local video was unmuted. We use this to calculate how
+         * much time we as a sender have had to ramp-up.
+         */
+        this._timeVideoUnmuted = -1;
+
+
+        // We assume a global startBitrate value for the sake of simplicity.
+        if (options.startBitrate && options.startBitrate > 0) {
+            startBitrate = options.startBitrate;
+        }
+
+        // TODO: consider ignoring these events and letting the user of
+        // lib-jitsi-meet handle these separately.
+        conference.on(
+            ConferenceEvents.CONNECTION_INTERRUPTED,
+            () => {
+                this._updateLocalConnectionQuality(0);
+                this.eventEmitter.emit(
+                    ConnectionQualityEvents.LOCAL_STATS_UPDATED,
+                    this._localStats);
+                this._broadcastLocalStats();
+            });
+
+        conference.room.addListener(
+            XMPPEvents.ICE_CONNECTION_STATE_CHANGED,
+            (newState) => {
+                if (newState === 'connected') {
+                    this._timeIceConnected = window.performance.now();
+                }
+            });
+
+        // Listen to DataChannel message from other participants in the
+        // conference, and update the _remoteStats field accordingly.
+        conference.on(
+            ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
+            (participant, payload) => {
+                if (payload.type === STATS_MESSAGE_TYPE) {
+                    this._updateRemoteStats(
+                        participant.getId(), payload.values);
+                }
+        });
+
+        // Listen to local statistics events originating from the RTC module
+        // and update the _localStats field.
+        // Oh, and by the way, the resolutions of all remote participants are
+        // also piggy-backed in these "local" statistics. It's obvious, really,
+        // if one carefully reads the *code* (but not the docs) in
+        // UI/VideoLayout/VideoLayout.js#updateLocalConnectionStats in
+        // jitsi-meet
+        // TODO: We should keep track of the remote resolution in _remoteStats,
+        // and notify about changes via separate events.
+        conference.on(
+            ConferenceEvents.CONNECTION_STATS,
+            this._updateLocalStats.bind(this));
+
+        // Save the last time we were unmuted.
+        conference.on(
+            ConferenceEvents.TRACK_MUTE_CHANGED,
+            (track) => {
+                if (track.isVideoTrack()) {
+                    if (track.isMuted()) {
+                        this._timeVideoUnmuted = -1;
+                    } else {
+                        this._maybeUpdateUnmuteTime();
+                    }
+                }
+            });
+        conference.on(
+            ConferenceEvents.TRACK_ADDED,
+            (track) => {
+                if (track.isVideoTrack() && !track.isMuted())
+                {
+                    this._maybeUpdateUnmuteTime();
+                }
+            });
+    }
+
+    /**
+     * Sets _timeVideoUnmuted if it was previously unset. If it was already set,
+     * doesn't change it.
+     */
+    _maybeUpdateUnmuteTime() {
+        if (this._timeVideoUnmuted < 0) {
+            this._timeVideoUnmuted = window.performance.now();
+        }
+    }
+
+    /**
+     * Calculates a new "connection quality" value.
+     * @param videoType {VideoType} the type of the video source (camera or
+     * a screen capture).
+     * @param isMuted {boolean} whether the local video is muted.
+     * @param resolutionName {Resolution} the input resolution used by the
+     * camera.
+     * @returns {*} the newly calculated connection quality.
+     */
+    _calculateConnectionQuality(videoType, isMuted, resolutionName) {
+
+        // resolutionName is an index into Resolutions (where "720" is
+        // "1280x720" and "960" is "960x720" ...).
+        let resolution = Resolutions[resolutionName];
+
+        let quality = 100;
+
+        if (isMuted || !resolution || videoType === VideoType.DESKTOP
+            || this._timeIceConnected < 0
+            || this._timeVideoUnmuted < 0) {
+
+            // Calculate a value based on packet loss only.
+            if (!this._localStats.packetLoss
+                || this._localStats.packetLoss.total === undefined) {
+                logger.error("Cannot calculate connection quality, unknown "
+                    + "packet loss.");
+                quality = 100;
+            } else {
+                let loss = this._localStats.packetLoss.total;
+                if (loss <= 2) {
+                    quality = 100;
+                } else if (loss <= 4) {
+                    quality = 70; // 4 bars
+                } else if (loss <= 6) {
+                    quality = 50; // 3 bars
+                } else if (loss <= 8) {
+                    quality = 30; // 2 bars
+                } else if (loss <= 12) {
+                    quality = 10; // 1 bars
+                } else {
+                    quality = 0; // Still 1 bar, but slower climb-up.
+                }
+            }
+        } else {
+            // Calculate a value based on the sending bitrate.
+
+            // time since sending of video was enabled.
+            let millisSinceStart = window.performance.now()
+                    - Math.max(this._timeVideoUnmuted, this._timeIceConnected);
+
+            // expected sending bitrate in perfect conditions
+            let target
+                = getTarget(this._simulcast, resolution, millisSinceStart);
+            target = 0.9 * target;
+
+            quality = 100 * this._localStats.bitrate.upload / target;
+
+            // Whatever the bitrate, drop early if there is significant loss
+            if (this._localStats.packetLoss
+                && this._localStats.packetLoss.total >= 10) {
+                quality = Math.min(quality, 30);
+            }
+        }
+
+        // Make sure that the quality doesn't climb quickly
+        if (this._lastConnectionQualityUpdate > 0)
+        {
+            let maxIncreasePerSecond = 2;
+            let prevConnectionQuality = this._localStats.connectionQuality;
+            let diffSeconds
+                = (window.performance.now()
+                    - this._lastConnectionQualityUpdate) / 1000;
+            quality = Math.min(
+                quality,
+                prevConnectionQuality + diffSeconds * maxIncreasePerSecond);
+        }
+
+        return Math.min(100, quality);
+    }
+
+    /**
+     * Updates the localConnectionQuality value
+     * @param values {number} the new value. Should be in [0, 100].
+     */
+    _updateLocalConnectionQuality(value) {
+        this._localStats.connectionQuality = value;
+        this._lastConnectionQualityUpdate = window.performance.now();
+    }
+
+    /**
+     * Broadcasts the local statistics to all other participants in the
+     * conference.
+     */
+    _broadcastLocalStats() {
+        // Send only the data that remote participants care about.
+        let data = {
+            bitrate: this._localStats.bitrate,
+            packetLoss: this._localStats.packetLoss,
+            connectionQuality: this._localStats.connectionQuality
+        };
+
+        // TODO: It looks like the remote participants don't really "care"
+        // about the resolution, and they look at their local rendered
+        // resolution instead. Consider removing this.
+        let localVideoTrack
+            = this._conference.getLocalTracks(MediaType.VIDEO)
+                .find(track => track.isVideoTrack());
+        if (localVideoTrack && localVideoTrack.resolution) {
+            data.resolution = localVideoTrack.resolution;
+        }
+
+        try {
+            this._conference.broadcastEndpointMessage({
+                type: STATS_MESSAGE_TYPE,
+                values: data });
+        } catch (e) {
+            // We often hit this in the beginning of a call, before the data
+            // channel is ready. It is not a big problem, because we will
+            // send the statistics again after a few seconds, and the error is
+            // already logged elsewhere. So just ignore it.
+
+            //let errorMsg = "Failed to broadcast local stats";
+            //logger.error(errorMsg, e);
+            //GlobalOnErrorHandler.callErrorHandler(
+            //    new Error(errorMsg + ": " + e));
+        }
+    }
+
+    /**
+     * Updates the local statistics
+     * @param data new statistics
+     */
+    _updateLocalStats(data) {
+        let key;
+        let updateLocalConnectionQuality
+            = !this._conference.isConnectionInterrupted();
+        let localVideoTrack =
+                this._conference.getLocalTracks(MediaType.VIDEO)
+                    .find(track => track.isVideoTrack());
+        let videoType = localVideoTrack ? localVideoTrack.videoType : undefined;
+        let isMuted = localVideoTrack ? localVideoTrack.isMuted() : true;
+        let resolution = localVideoTrack ? localVideoTrack.resolution : null;
+
+        if (!isMuted) {
+            this._maybeUpdateUnmuteTime();
+        }
+
+        // Copy the fields already in 'data'.
+        for (key in data) {
+            if (data.hasOwnProperty(key)) {
+                this._localStats[key] = data[key];
+            }
+        }
+
+        // And re-calculate the connectionQuality field.
+        if (updateLocalConnectionQuality) {
+            this._updateLocalConnectionQuality(
+                this._calculateConnectionQuality(
+                    videoType,
+                    isMuted,
+                    resolution));
+        }
+
+        this.eventEmitter.emit(
+            ConnectionQualityEvents.LOCAL_STATS_UPDATED,
+            this._localStats);
+        this._broadcastLocalStats();
+    }
+
+    /**
+     * Updates remote statistics
+     * @param id the id of the remote participant
+     * @param data the statistics received
+     */
+    _updateRemoteStats(id, data) {
+            // Use only the fields we need
+            this._remoteStats[id] = {
+                bitrate: data.bitrate,
+                packetLoss: data.packetLoss,
+                connectionQuality: data.connectionQuality
+            };
+
+            this.eventEmitter.emit(
+                ConnectionQualityEvents.REMOTE_STATS_UPDATED,
+                id,
+                this._remoteStats[id]);
+    }
+
+    /**
+     * Returns the local statistics.
+     * Exported only for use in jitsi-meet-torture.
+     */
+    getStats() {
+        return this._localStats;
+    }
+}

modules/xmpp/JingleSessionPC.js

@@ -142,6 +142,9 @@ JingleSessionPC.prototype.doInitialize = function () {
                     ":\t", now);
         Statistics.analytics.sendEvent(
             'ice.' + self.peerconnection.iceConnectionState, {value: now});
+        self.room.eventEmitter.emit(
+            XMPPEvents.ICE_CONNECTION_STATE_CHANGED,
+            self.peerconnection.iceConnectionState);
         switch (self.peerconnection.iceConnectionState) {
             case 'connected':
 
@@ -813,7 +816,7 @@ JingleSessionPC.prototype._modifySources = function (successCallback, queueCallb
      * optional error through (1) logger, (2) GlobalOnErrorHandler, and (3)
      * queueCallback.
      *
-     * @param {string} errmsg the error messsage to report
+     * @param {string} errmsg the error message to report
      * @param {*} error an optional error to report in addition to errmsg
      */
     function reportError(errmsg, err) {

service/connectionquality/CQEvents.js

@@ -1,7 +0,0 @@
-var CQEvents = {
-    LOCALSTATS_UPDATED: "cq.localstats_updated",
-    REMOTESTATS_UPDATED: "cq.remotestats_updated",
-    STOP: "cq.stop"
-};
-
-module.exports = CQEvents;

service/connectivity/ConnectionQualityEvents.js

@@ -0,0 +1,10 @@
+/**
+ * Indicates that the local connection statistics were updated.
+ */
+export const LOCAL_STATS_UPDATED = "cq.local_stats_updated";
+
+/**
+ * Indicates that the connection statistics for a particular remote participant
+ * were updated.
+ */
+export const REMOTE_STATS_UPDATED = "cq.remote_stats_updated";

service/xmpp/XMPPEvents.js

@@ -183,6 +183,9 @@ var XMPPEvents = {
     LOCAL_UFRAG_CHANGED: "xmpp.local_ufrag_changed",
     // Designates an event indicating that the local ICE username fragment of
     // the jingle session has changed.
-    REMOTE_UFRAG_CHANGED: "xmpp.remote_ufrag_changed"
+    REMOTE_UFRAG_CHANGED: "xmpp.remote_ufrag_changed",
+    // Designates an event indicating that the local ICE connection state has
+    // changed.
+    ICE_CONNECTION_STATE_CHANGED: "xmpp.ice_connection_state_changed"
 };
 module.exports = XMPPEvents;