Merge pull request #259 from jitsi/participant_conn_status

Adds participant connection status notifications

JitsiConference.js

@@ -18,6 +18,8 @@ var GlobalOnErrorHandler = require("./modules/util/GlobalOnErrorHandler");
 var JitsiConferenceEventManager = require("./JitsiConferenceEventManager");
 var VideoType = require('./service/RTC/VideoType');
 var Transcriber = require("./modules/transcription/transcriber");
+var ParticipantConnectionStatus
+    = require("./modules/connectivity/ParticipantConnectionStatus");
 
 /**
  * Creates a JitsiConference object with the given name and properties.
@@ -92,6 +94,10 @@ JitsiConference.prototype._init = function (options) {
         this.eventManager.setupRTCListeners();
     }
 
+    this.participantConnectionStatus
+        = new ParticipantConnectionStatus(this.rtc, this);
+    this.participantConnectionStatus.init();
+
     if(!this.statistics) {
         this.statistics = new Statistics(this.xmpp, {
             callStatsID: this.options.config.callStatsID,
@@ -151,6 +157,11 @@ JitsiConference.prototype._leaveRoomAndRemoveParticipants = function () {
 JitsiConference.prototype.leave = function () {
     var conference = this;
 
+    if (this.participantConnectionStatus) {
+        this.participantConnectionStatus.dispose();
+        this.participantConnectionStatus = null;
+    }
+
     this.statistics.stopCallStats();
     this.rtc.closeAllDataChannels();
 

JitsiConferenceEvents.js

@@ -81,6 +81,16 @@ export const LOCK_STATE_CHANGED = "conference.lock_state_changed";
  * New text message was received.
  */
 export const MESSAGE_RECEIVED = "conference.messageReceived";
+/**
+ * Event fired when JVB sends notification about interrupted/restored user's
+ * ICE connection status. First argument is the ID of the participant and
+ * the seconds is a boolean indicating if the connection is currently
+ * active(true = active, false = interrupted).
+ * The current status value can be obtained by calling
+ * JitsiParticipant.isConnectionActive().
+ */
+export const PARTICIPANT_CONN_STATUS_CHANGED
+    = "conference.participant_conn_status_changed";
 /**
  * Indicates that a the value of a specific property of a specific participant
  * has changed.

JitsiParticipant.js

@@ -30,6 +30,7 @@ export default class JitsiParticipant {
             video: undefined
         };
         this._hidden = hidden;
+        this._isConnectionActive = true;
         this._properties = {};
     }
 
@@ -48,6 +49,26 @@ export default class JitsiParticipant {
         return this._properties[name];
     }
 
+    /**
+     * Updates participant's connection status.
+     * @param {boolean} isActive true if the user's connection is fine or false
+     * when the user is having connectivity issues.
+     * @private
+     */
+    _setIsConnectionActive(isActive) {
+        this._isConnectionActive = isActive;
+    }
+
+    /**
+     * Checks participant's connectivity status.
+     *
+     * @returns {boolean} true if the connection is currently ok or false when
+     * the user is having connectivity issues.
+     */
+    isConnectionActive() {
+        return this._isConnectionActive;
+    }
+
     /**
      * Sets the value of a property of this participant, and fires an event if
      * the value has changed.

modules/RTC/DataChannels.js

@@ -143,6 +143,14 @@ DataChannels.prototype.onDataChannel = function (event) {
                     RTCEvents.ENDPOINT_MESSAGE_RECEIVED, obj.from,
                     obj.msgPayload);
             }
+            else if ("EndpointConnectivityStatusChangeEvent" === colibriClass) {
+                var endpoint = obj.endpoint;
+                var isActive = obj.active === "true";
+                logger.info("Endpoint connection status changed: " + endpoint
+                           + " active ? " + isActive);
+                self.eventEmitter.emit(RTCEvents.ENDPOINT_CONN_STATUS_CHANGED,
+                    endpoint, isActive);
+            }
             else {
                 logger.debug("Data channel JSON-formatted message: ", obj);
                 // The received message appears to be appropriately formatted

modules/RTC/JitsiLocalTrack.js

@@ -340,7 +340,7 @@ JitsiLocalTrack.prototype._setMute = function (mute) {
             return self._sendMuteStatus(mute);
         })
         .then(function() {
-            self.eventEmitter.emit(JitsiTrackEvents.TRACK_MUTE_CHANGED);
+            self.eventEmitter.emit(JitsiTrackEvents.TRACK_MUTE_CHANGED, this);
         });
 };
 

modules/RTC/JitsiRemoteTrack.js

@@ -1,6 +1,10 @@
+/* global __filename, module */
+var logger = require("jitsi-meet-logger").getLogger(__filename);
+
 var JitsiTrack = require("./JitsiTrack");
 import * as JitsiTrackEvents from "../../JitsiTrackEvents";
 var RTCBrowserType = require("./RTCBrowserType");
+var RTCEvents = require("../../service/RTC/RTCEvents");
 var Statistics = require("../statistics/statistics");
 var AdapterJS = require("./adapter.screenshare");
 
@@ -9,7 +13,7 @@ var ttfmTrackerVideoAttached = false;
 
 /**
  * Represents a single media track (either audio or video).
- * @param RTC the rtc instance.
+ * @param rtc {RTC} the RTC service instance.
  * @param ownerJid the MUC JID of the track owner
  * @param stream WebRTC MediaStream, parent of the track
  * @param track underlying WebRTC MediaStreamTrack for new JitsiRemoteTrack
@@ -19,22 +23,52 @@ var ttfmTrackerVideoAttached = false;
  * @param muted intial muted state of the JitsiRemoteTrack
  * @constructor
  */
-function JitsiRemoteTrack(conference, ownerJid, stream, track, mediaType, videoType,
+function JitsiRemoteTrack(rtc, conference, ownerJid, stream, track, mediaType, videoType,
                           ssrc, muted) {
     JitsiTrack.call(
         this, conference, stream, track, function () {}, mediaType, videoType, ssrc);
-    this.conference = conference;
+    this.rtc = rtc;
     this.peerjid = ownerJid;
     this.muted = muted;
     // we want to mark whether the track has been ever muted
     // to detect ttfm events for startmuted conferences, as it can significantly
     // increase ttfm values
     this.hasBeenMuted = muted;
+    // Bind 'onmute' and 'onunmute' event handlers
+    if (this.rtc && this.track)
+        this._bindMuteHandlers();
 }
 
 JitsiRemoteTrack.prototype = Object.create(JitsiTrack.prototype);
 JitsiRemoteTrack.prototype.constructor = JitsiRemoteTrack;
 
+JitsiRemoteTrack.prototype._bindMuteHandlers = function() {
+    // Bind 'onmute'
+    // FIXME it would be better to use recently added '_setHandler' method, but
+    // 1. It does not allow to set more than one handler to the event
+    // 2. It does mix MediaStream('inactive') with MediaStreamTrack events
+    // 3. Allowing to bind more than one event handler requires too much
+    //    refactoring around camera issues detection.
+    this.track.addEventListener('mute', function () {
+
+        logger.debug(
+            '"onmute" event(' + Date.now() + '): ',
+            this.getParticipantId(), this.getType(), this.getSSRC());
+
+        this.rtc.eventEmitter.emit(RTCEvents.REMOTE_TRACK_MUTE, this);
+    }.bind(this));
+
+    // Bind 'onunmute'
+    this.track.addEventListener('unmute', function () {
+
+        logger.debug(
+            '"onunmute" event(' + Date.now() + '): ',
+            this.getParticipantId(), this.getType(), this.getSSRC());
+
+        this.rtc.eventEmitter.emit(RTCEvents.REMOTE_TRACK_UNMUTE, this);
+    }.bind(this));
+};
+
 /**
  * Sets current muted status and fires an events for the change.
  * @param value the muted status.
@@ -51,7 +85,7 @@ JitsiRemoteTrack.prototype.setMute = function (value) {
         this.stream.muted = value;
 
     this.muted = value;
-    this.eventEmitter.emit(JitsiTrackEvents.TRACK_MUTE_CHANGED);
+    this.eventEmitter.emit(JitsiTrackEvents.TRACK_MUTE_CHANGED, this);
 };
 
 /**

modules/RTC/JitsiTrack.js

@@ -149,6 +149,15 @@ JitsiTrack.prototype.isVideoTrack = function () {
     return this.getType() === MediaType.VIDEO;
 };
 
+/**
+ * Checks whether this is a local track.
+ * @abstract
+ * @return {boolean} 'true' if it's a local track or 'false' otherwise.
+ */
+JitsiTrack.prototype.isLocal = function () {
+    throw new Error("Not implemented by subclass");
+};
+
 /**
  * Returns the WebRTC MediaStream instance.
  */

modules/RTC/RTC.js

@@ -301,7 +301,7 @@ RTC.prototype.removeLocalTrack = function (track) {
 RTC.prototype.createRemoteTrack = function (event) {
     var ownerJid = event.owner;
     var remoteTrack = new JitsiRemoteTrack(
-        this.conference,  ownerJid, event.stream,    event.track,
+        this, this.conference, ownerJid, event.stream, event.track,
         event.mediaType, event.videoType, event.ssrc, event.muted);
     var resource = Strophe.getResourceFromJid(ownerJid);
     var remoteTracks

modules/RTC/RTCBrowserType.js

@@ -106,6 +106,16 @@ var RTCBrowserType = {
         return RTCBrowserType.isIExplorer() || RTCBrowserType.isSafari();
     },
 
+    /**
+     * Checks if the current browser triggers 'onmute'/'onunmute' events when
+     * user's connection is interrupted and the video stops playback.
+     * @returns {*|boolean} 'true' if the event is supported or 'false'
+     * otherwise.
+     */
+    isVideoMuteOnConnInterruptedSupported: function () {
+        return RTCBrowserType.isChrome();
+    },
+
     /**
      * Returns Firefox version.
      * @returns {number|null}

modules/connectivity/ParticipantConnectionStatus.js

@@ -0,0 +1,306 @@
+/* global __filename, module, require */
+var logger = require('jitsi-meet-logger').getLogger(__filename);
+var MediaType = require('../../service/RTC/MediaType');
+var RTCBrowserType = require('../RTC/RTCBrowserType');
+var RTCEvents = require('../../service/RTC/RTCEvents');
+
+import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
+import * as JitsiTrackEvents from '../../JitsiTrackEvents';
+
+/**
+ * How long we're going to wait after the RTC video track muted event for
+ * the corresponding signalling mute event, before the connection interrupted
+ * is fired.
+ *
+ * @type {number} amount of time in milliseconds
+ */
+const RTC_MUTE_TIMEOUT = 1000;
+
+/**
+ * Class is responsible for emitting
+ * JitsiConferenceEvents.PARTICIPANT_CONN_STATUS_CHANGED events.
+ *
+ * @constructor
+ * @param rtc {RTC} the RTC service instance
+ * @param conference {JitsiConference} parent conference instance
+ */
+function ParticipantConnectionStatus(rtc, conference) {
+    this.rtc = rtc;
+    this.conference = conference;
+    /**
+     * A map of the "endpoint ID"(which corresponds to the resource part of MUC
+     * JID(nickname)) to the timeout callback IDs scheduled using
+     * window.setTimeout.
+     * @type {Object.<string, number>}
+     */
+    this.trackTimers = {};
+}
+
+/**
+ * Initializes <tt>ParticipantConnectionStatus</tt> and bind required event
+ * listeners.
+ */
+ParticipantConnectionStatus.prototype.init = function() {
+
+    this._onEndpointConnStatusChanged
+        = this.onEndpointConnStatusChanged.bind(this);
+
+    this.rtc.addListener(
+        RTCEvents.ENDPOINT_CONN_STATUS_CHANGED,
+        this._onEndpointConnStatusChanged);
+
+    // On some browsers MediaStreamTrack trigger "onmute"/"onunmute"
+    // events for video type tracks when they stop receiving data which is
+    // often a sign that remote user is having connectivity issues
+    if (RTCBrowserType.isVideoMuteOnConnInterruptedSupported()) {
+
+        this._onTrackRtcMuted = this.onTrackRtcMuted.bind(this);
+        this.rtc.addListener(
+            RTCEvents.REMOTE_TRACK_MUTE, this._onTrackRtcMuted);
+
+        this._onTrackRtcUnmuted = this.onTrackRtcUnmuted.bind(this);
+        this.rtc.addListener(
+            RTCEvents.REMOTE_TRACK_UNMUTE, this._onTrackRtcUnmuted);
+
+        // Track added/removed listeners are used to bind "mute"/"unmute"
+        // event handlers
+        this._onRemoteTrackAdded = this.onRemoteTrackAdded.bind(this);
+        this.conference.on(
+            JitsiConferenceEvents.TRACK_ADDED, this._onRemoteTrackAdded);
+        this._onRemoteTrackRemoved = this.onRemoteTrackRemoved.bind(this);
+        this.conference.on(
+            JitsiConferenceEvents.TRACK_REMOVED, this._onRemoteTrackRemoved);
+
+        // Listened which will be bound to JitsiRemoteTrack to listen for
+        // signalling mute/unmute events.
+        this._onSignallingMuteChanged = this.onSignallingMuteChanged.bind(this);
+    }
+};
+
+/**
+ * Removes all event listeners and disposes of all resources held by this
+ * instance.
+ */
+ParticipantConnectionStatus.prototype.dispose = function () {
+
+    this.rtc.removeListener(
+        RTCEvents.ENDPOINT_CONN_STATUS_CHANGED,
+        this._onEndpointConnStatusChanged);
+
+    if (RTCBrowserType.isVideoMuteOnConnInterruptedSupported()) {
+        this.rtc.removeListener(
+            RTCEvents.REMOTE_TRACK_MUTE, this._onTrackRtcMuted);
+        this.rtc.removeListener(
+            RTCEvents.REMOTE_TRACK_UNMUTE, this._onTrackRtcUnmuted);
+        this.conference.off(
+            JitsiConferenceEvents.TRACK_ADDED, this._onRemoteTrackAdded);
+        this.conference.off(
+            JitsiConferenceEvents.TRACK_REMOVED, this._onRemoteTrackRemoved);
+    }
+
+    Object.keys(this.trackTimers).forEach(function (participantId) {
+        this.clearTimeout(participantId);
+    }.bind(this));
+};
+
+/**
+ * Checks whether given <tt>JitsiParticipant</tt> has any muted video
+ * <tt>MediaStreamTrack</tt>s.
+ *
+ * @param {JitsiParticipant} participant to be checked for muted video tracks
+ *
+ * @return {boolean} <tt>true</tt> if given <tt>participant</tt> contains any
+ * video <tt>MediaStreamTrack</tt>s muted according to their 'muted' field.
+ */
+var hasRtcMutedVideoTrack = function (participant) {
+    return participant.getTracks().some(function(jitsiTrack) {
+        var rtcTrack = jitsiTrack.getTrack();
+        return jitsiTrack.getType() === MediaType.VIDEO
+            && rtcTrack && rtcTrack.muted === true;
+    });
+};
+
+/**
+ * Handles RTCEvents.ENDPOINT_CONN_STATUS_CHANGED triggered when we receive
+ * notification over the data channel from the bridge about endpoint's
+ * connection status update.
+ * @param endpointId {string} the endpoint ID(MUC nickname/resource JID)
+ * @param isActive {boolean} true if the connection is OK or false otherwise
+ */
+ParticipantConnectionStatus.prototype.onEndpointConnStatusChanged
+= function(endpointId, isActive) {
+
+    logger.debug(
+        'Detector RTCEvents.ENDPOINT_CONN_STATUS_CHANGED('
+            + Date.now() +'): ' + endpointId + ': ' + isActive);
+
+    // Filter out events for the local JID for now
+    if (endpointId !== this.conference.myUserId()) {
+        var participant = this.conference.getParticipantById(endpointId);
+        // Delay the 'active' event until the video track gets RTC unmuted event
+        if (isActive
+                && RTCBrowserType.isVideoMuteOnConnInterruptedSupported()
+                && participant
+                && hasRtcMutedVideoTrack(participant)
+                && !participant.isVideoMuted()) {
+            logger.debug(
+                'Ignoring RTCEvents.ENDPOINT_CONN_STATUS_CHANGED -'
+                    + ' will wait for unmute event');
+        } else {
+            this._changeConnectionStatus(endpointId, isActive);
+        }
+    }
+};
+
+ParticipantConnectionStatus.prototype._changeConnectionStatus
+= function (endpointId, newStatus) {
+    var participant = this.conference.getParticipantById(endpointId);
+    if (!participant) {
+        // This will happen when participant exits the conference with broken
+        // ICE connection and we join after that. The bridge keeps sending
+        // that notification until the conference does not expire.
+        logger.warn(
+            'Missed participant connection status update - ' +
+                'no participant for endpoint: ' + endpointId);
+        return;
+    }
+    if (participant.isConnectionActive() !== newStatus) {
+        participant._setIsConnectionActive(newStatus);
+        logger.debug(
+            'Emit endpoint conn status(' + Date.now() + '): ',
+            endpointId, newStatus);
+        this.conference.eventEmitter.emit(
+            JitsiConferenceEvents.PARTICIPANT_CONN_STATUS_CHANGED,
+            endpointId, newStatus);
+    }
+};
+
+/**
+ * Reset the postponed "connection interrupted" event which was previously
+ * scheduled as a timeout on RTC 'onmute' event.
+ *
+ * @param participantId the participant for which the "connection interrupted"
+ * timeout was scheduled
+ */
+ParticipantConnectionStatus.prototype.clearTimeout = function (participantId) {
+    if (this.trackTimers[participantId]) {
+        window.clearTimeout(this.trackTimers[participantId]);
+        this.trackTimers[participantId] = null;
+    }
+};
+
+/**
+ * Bind signalling mute event listeners for video {JitsiRemoteTrack} when
+ * a new one is added to the conference.
+ *
+ * @param {JitsiTrack} remoteTrack the {JitsiTrack} which is being added to
+ * the conference.
+ */
+ParticipantConnectionStatus.prototype.onRemoteTrackAdded
+= function(remoteTrack) {
+    if (!remoteTrack.isLocal() && remoteTrack.getType() === MediaType.VIDEO) {
+
+        logger.debug(
+            'Detector on remote track added: ', remoteTrack.getParticipantId());
+
+        remoteTrack.on(
+            JitsiTrackEvents.TRACK_MUTE_CHANGED,
+            this._onSignallingMuteChanged);
+    }
+};
+
+/**
+ * Removes all event listeners bound to the remote video track and clears any
+ * related timeouts.
+ *
+ * @param {JitsiRemoteTrack} remoteTrack the remote track which is being removed
+ * from the conference.
+ */
+ParticipantConnectionStatus.prototype.onRemoteTrackRemoved
+= function(remoteTrack) {
+    if (!remoteTrack.isLocal() && remoteTrack.getType() === MediaType.VIDEO) {
+        logger.debug(
+            'Detector on remote track removed: ',
+            remoteTrack.getParticipantId());
+        remoteTrack.off(
+            JitsiTrackEvents.TRACK_MUTE_CHANGED,
+            this._onSignallingMuteChanged);
+        this.clearTimeout(remoteTrack.getParticipantId());
+    }
+};
+
+/**
+ * Handles RTC 'onmute' event for the video track.
+ *
+ * @param track {JitsiRemoteTrack} the video track for which 'onmute' event will
+ * be processed.
+ */
+ParticipantConnectionStatus.prototype.onTrackRtcMuted = function(track) {
+    var participantId = track.getParticipantId();
+    var participant = this.conference.getParticipantById(participantId);
+    logger.debug('Detector track RTC muted: ', participantId);
+    if (!participant) {
+        logger.error('No participant for id: ' + participantId);
+        return;
+    }
+    if (!participant.isVideoMuted()) {
+        // If the user is not muted according to the signalling we'll give it
+        // some time, before the connection interrupted event is triggered.
+        this.trackTimers[participantId] = window.setTimeout(function () {
+            if (!track.isMuted() && participant.isConnectionActive()) {
+                logger.info(
+                    'Connection interrupted through the RTC mute: '
+                        + participantId, Date.now());
+                this._changeConnectionStatus(participantId, false);
+            }
+            this.clearTimeout(participantId);
+        }.bind(this), RTC_MUTE_TIMEOUT);
+    }
+};
+
+/**
+ * Handles RTC 'onunmute' event for the video track.
+ *
+ * @param track {JitsiRemoteTrack} the video track for which 'onunmute' event
+ * will be processed.
+ */
+ParticipantConnectionStatus.prototype.onTrackRtcUnmuted = function(track) {
+    logger.debug('Detector track RTC unmuted: ', track);
+    var participantId = track.getParticipantId();
+    if (!track.isMuted() &&
+        !this.conference.getParticipantById(participantId)
+            .isConnectionActive()) {
+        logger.info(
+            'Detector connection restored through the RTC unmute: '
+                + participantId, Date.now());
+        this._changeConnectionStatus(participantId, true);
+    }
+    this.clearTimeout(participantId);
+};
+
+/**
+ * Here the signalling "mute"/"unmute" events are processed.
+ *
+ * @param track {JitsiRemoteTrack} the remote video track for which
+ * the signalling mute/unmute event will be processed.
+ */
+ParticipantConnectionStatus.prototype.onSignallingMuteChanged
+= function (track) {
+    logger.debug(
+        'Detector on track signalling mute changed: ', track, track.isMuted());
+    var isMuted = track.isMuted();
+    var participantId = track.getParticipantId();
+    var participant = this.conference.getParticipantById(participantId);
+    if (!participant) {
+        logger.error('No participant for id: ' + participantId);
+        return;
+    }
+    var isConnectionActive = participant.isConnectionActive();
+    if (isMuted && isConnectionActive && this.trackTimers[participantId]) {
+        logger.debug(
+            'Signalling got in sync - cancelling task for: ' + participantId);
+        this.clearTimeout(participantId);
+    }
+};
+
+module.exports = ParticipantConnectionStatus;

service/RTC/RTCEvents.js

@@ -1,11 +1,14 @@
 var RTCEvents = {
     RTC_READY: "rtc.ready",
     DATA_CHANNEL_OPEN: "rtc.data_channel_open",
+    ENDPOINT_CONN_STATUS_CHANGED: "rtc.endpoint_conn_status_changed",
     LASTN_CHANGED: "rtc.lastn_changed",
     DOMINANTSPEAKER_CHANGED: "rtc.dominantspeaker_changed",
     LASTN_ENDPOINT_CHANGED: "rtc.lastn_endpoint_changed",
     AVAILABLE_DEVICES_CHANGED: "rtc.available_devices_changed",
     TRACK_ATTACHED: "rtc.track_attached",
+    REMOTE_TRACK_MUTE: "rtc.remote_track_mute",
+    REMOTE_TRACK_UNMUTE: "rtc.remote_track_unmute",
     AUDIO_OUTPUT_DEVICE_CHANGED: "rtc.audio_output_device_changed",
     DEVICE_LIST_CHANGED: "rtc.device_list_changed",
     DEVICE_LIST_AVAILABLE: "rtc.device_list_available",