feat(video-quality): control the sender resolution based on video quality settings (#1119)

* feat(video-quality): control the sender resolution based on video quality settings

* fix(video-quality): Apply the settings on newly created p2p/jvb jingle sessions
If a p2p/jvb session is not present when setSenderVideoConstraint is called,
make sure the settings are applied when they are created

JitsiConference.js

@@ -235,6 +235,12 @@ export default function JitsiConference(options) {
     this.recordingManager = new RecordingManager(this.room);
     this._conferenceJoinAnalyticsEventSent = false;
 
+    /**
+     * Max frame height that the user prefers to send to the remote participants.
+     * @type {number}
+     */
+    this.maxFrameHeight = null;
+
     if (browser.supportsInsertableStreams()) {
         this._e2eeCtx = new E2EEContext({ salt: this.options.name });
     }
@@ -1879,6 +1885,13 @@ JitsiConference.prototype._acceptJvbIncomingCall = function(
                 // to be turned off here.
                 if (this.isP2PActive() && this.jvbJingleSession) {
                     this._suspendMediaTransferForJvbConnection();
+                } else if (this.jvbJingleSession && this.maxFrameHeight) {
+                    // Apply user preferred max frame height if it was called before this
+                    // jingle session was created.
+                    this.jvbJingleSession.setSenderVideoConstraint(this.maxFrameHeight)
+                        .catch(err => {
+                            logger.error(`Sender video constraints failed on jvb session - ${err}`);
+                        });
                 }
 
                 // Setup E2EE.
@@ -2644,6 +2657,15 @@ JitsiConference.prototype._acceptP2PIncomingCall = function(
         () => {
             logger.debug('Got RESULT for P2P "session-accept"');
 
+            // Apply user preferred max frame height if it was called before this
+            // jingle session was created.
+            if (this.pendingVideoConstraintsOnP2P) {
+                this.p2pJingleSession.setSenderVideoConstraint(this.maxFrameHeight)
+                    .catch(err => {
+                        logger.error(`Sender video constraints failed on p2p session - ${err}`);
+                    });
+            }
+
             // Setup E2EE.
             for (const track of localTracks) {
                 this._setupSenderE2EEForTrack(jingleSession, track);
@@ -3261,6 +3283,32 @@ JitsiConference.prototype.setReceiverVideoConstraint = function(
     this.rtc.setReceiverVideoConstraint(maxFrameHeight);
 };
 
+/**
+ * Sets the maximum video size the local participant should send to remote
+ * participants.
+ * @param {number} maxFrameHeight - The user preferred max frame height.
+ * @returns {Promise} promise that will be resolved when the operation is
+ * successful and rejected otherwise.
+ */
+JitsiConference.prototype.setSenderVideoConstraint = function(maxFrameHeight) {
+    this.maxFrameHeight = maxFrameHeight;
+    this.pendingVideoConstraintsOnP2P = true;
+    const promises = [];
+
+    // We have to always set the sender video constraints on the jvb connection
+    // when we switch from p2p to jvb connection since we need to check if the
+    // tracks constraints have been modified when in p2p.
+    if (this.jvbJingleSession) {
+        promises.push(this.jvbJingleSession.setSenderVideoConstraint(maxFrameHeight));
+    }
+    if (this.p2pJingleSession) {
+        this.pendingVideoConstraintsOnP2P = false;
+        promises.push(this.p2pJingleSession.setSenderVideoConstraint(maxFrameHeight));
+    }
+
+    return Promise.all(promises);
+};
+
 /**
  * Creates a video SIP GW session and returns it if service is enabled. Before
  * creating a session one need to check whether video SIP GW service is

doc/API.md

@@ -406,7 +406,9 @@ Throws NetworkError or InvalidStateError or Error if the operation fails.
 34. setReceiverVideoConstraint(resolution) - set the desired resolution to get from JVB (180, 360, 720, 1080, etc).
     You should use that method if you are using simulcast.
 
-35. isHidden - checks if local user has joined as a "hidden" user. This is a specialized role used for integrations.
+35. setSenderVideoConstraint(resolution) - set the desired resolution to send to JVB or the peer (180, 360, 720).
+
+36. isHidden - checks if local user has joined as a "hidden" user. This is a specialized role used for integrations.
 
 JitsiTrack
 ======

modules/RTC/JitsiLocalTrack.js

@@ -551,7 +551,7 @@ export default class JitsiLocalTrack extends JitsiTrack {
                     = RTCUtils.obtainAudioAndVideoPermissions(streamOptions);
             }
 
-            promise.then(streamsInfo => {
+            promise = promise.then(streamsInfo => {
                 // The track kind for presenter track is video as well.
                 const mediaType = this.getType() === MediaType.PRESENTER ? MediaType.VIDEO : this.getType();
                 const streamInfo

modules/RTC/TPCUtils.js

@@ -4,6 +4,7 @@ import transform from 'sdp-transform';
 import * as JitsiTrackEvents from '../../JitsiTrackEvents';
 import browser from '../browser';
 import RTCEvents from '../../service/RTC/RTCEvents';
+import * as VideoType from '../../service/RTC/VideoType';
 
 const logger = getLogger(__filename);
 const SIM_LAYER_1_RID = '1';
@@ -47,6 +48,12 @@ export class TPCUtils {
                 scaleResolutionDownBy: browser.isFirefox() ? 4.0 : 1.0
             }
         ];
+
+        /**
+         * Resolution height constraints for the simulcast encodings that
+         * are configured for the video tracks.
+         */
+        this.simulcastStreamConstraints = [];
     }
 
     /**
@@ -168,6 +175,25 @@ export class TPCUtils {
         });
     }
 
+    /**
+     * Constructs resolution height constraints for the simulcast encodings that are
+     * created for a given local video track.
+     * @param {MediaStreamTrack} track - the local video track.
+     * @returns {void}
+     */
+    _setSimulcastStreamConstraints(track) {
+        const height = track.getSettings().height;
+
+        for (const encoding in this.simulcastEncodings) {
+            if (this.simulcastEncodings.hasOwnProperty(encoding)) {
+                this.simulcastStreamConstraints.push({
+                    height: height / this.simulcastEncodings[encoding].scaleResolutionDownBy,
+                    rid: this.simulcastEncodings[encoding].rid
+                });
+            }
+        }
+    }
+
     /**
     * Adds {@link JitsiLocalTrack} to the WebRTC peerconnection for the first time.
     * @param {JitsiLocalTrack} track - track to be added to the peerconnection.
@@ -196,6 +222,11 @@ export class TPCUtils {
             // unused "recv-only" transceiver.
             this.pc.peerconnection.addTrack(track);
         }
+
+        // Construct the simulcast stream constraints for the newly added track.
+        if (localTrack.isVideoTrack() && localTrack.videoType === VideoType.CAMERA && this.pc.isSimulcastOn()) {
+            this._setSimulcastStreamConstraints(localTrack.getTrack());
+        }
     }
 
     /**

modules/RTC/TraceablePeerConnection.js

@@ -6,7 +6,12 @@ import transform from 'sdp-transform';
 
 import * as GlobalOnErrorHandler from '../util/GlobalOnErrorHandler';
 import JitsiRemoteTrack from './JitsiRemoteTrack';
+import {
+    TRACK_ADDED,
+    TRACK_MUTE_CHANGED
+} from '../../JitsiConferenceEvents';
 import * as MediaType from '../../service/RTC/MediaType';
+import * as VideoType from '../../service/RTC/VideoType';
 import LocalSdpMunger from './LocalSdpMunger';
 import RTC from './RTC';
 import RTCUtils from './RTCUtils';
@@ -335,6 +340,29 @@ export default function TraceablePeerConnection(
         }, 1000);
     }
 
+    // Set sender video constraints when a new local video track is added
+    // to the conference or when it is unmuted.
+    this.senderVideoMaxHeight = null;
+    const maybeSetSenderVideoConstraints = track => {
+        if (track.isLocal()
+            && !track.isMuted()
+            && track.isVideoTrack()
+            && track.videoType === VideoType.CAMERA
+            && this.senderVideoMaxHeight) {
+            this.setSenderVideoConstraint(this.senderVideoMaxHeight)
+                .catch(err => {
+                    logger.error(`Settings sender video constraints failed: ${err}`);
+                });
+        }
+    };
+
+    this.rtc.conference.on(
+        TRACK_ADDED,
+        maybeSetSenderVideoConstraints);
+    this.rtc.conference.on(
+        TRACK_MUTE_CHANGED,
+        maybeSetSenderVideoConstraints);
+
     logger.info(`Create new ${this}`);
 }
 
@@ -1469,6 +1497,11 @@ TraceablePeerConnection.prototype.addTrack = function(track, isInitiator = false
     if (browser.usesUnifiedPlan() && !browser.usesSdpMungingForSimulcast()) {
         this.tpcUtils.setEncodings(track);
     }
+
+    // Construct the simulcast stream constraints for the newly added track.
+    if (track.isVideoTrack() && track.videoType === VideoType.CAMERA && this.isSimulcastOn()) {
+        this.tpcUtils._setSimulcastStreamConstraints(track.getTrack());
+    }
 };
 
 /**
@@ -2042,6 +2075,79 @@ TraceablePeerConnection.prototype.setRemoteDescription = function(description) {
     });
 };
 
+/**
+ * Changes the resolution of the video stream that is sent to the peer based on
+ * the user preferred value. If simulcast is enabled on the peerconection, all the
+ * simulcast encodings that have a resolution height lower or equal to the value
+ * provided will remain active. For the non-simulcast case, video constraint is
+ * applied on the track.
+ * @param {number} frameHeight - The user preferred max frame height.
+ * @returns {Promise} promise that will be resolved when the operation is
+ * successful and rejected otherwise.
+ */
+TraceablePeerConnection.prototype.setSenderVideoConstraint = function(frameHeight) {
+    this.senderVideoMaxHeight = frameHeight;
+    const localVideoTrack = Array.from(this.localTracks.values()).find(t => t.isVideoTrack());
+
+    if (!localVideoTrack || localVideoTrack.isMuted() || localVideoTrack.videoType !== VideoType.CAMERA) {
+        return Promise.resolve();
+    }
+    const track = localVideoTrack.getTrack();
+
+    if (this.isSimulcastOn()) {
+        let promise = Promise.resolve();
+
+        // Check if the track constraints have been modified in p2p mode, apply
+        // the constraints that were used for creating the track if that is the case.
+        const height = localVideoTrack._constraints.height.ideal
+            ? localVideoTrack._constraints.height.ideal
+            : localVideoTrack._constraints.height;
+
+        if (track.getSettings().height !== height) {
+            promise = track.applyConstraints(localVideoTrack._constraints);
+        }
+
+        return promise
+            .then(() => {
+                // Determine the encodings that need to stay enabled based on the
+                // new frameHeight provided.
+                const encodingsEnabledState = this.tpcUtils.simulcastStreamConstraints
+                    .map(constraint => constraint.height <= frameHeight);
+                const videoSender = this.findSenderByKind(MediaType.VIDEO);
+
+                if (!videoSender) {
+                    return Promise.reject(new Error('RTCRtpSender not found for local video'));
+                }
+                const parameters = videoSender.getParameters();
+
+                if (!parameters || !parameters.encodings || !parameters.encodings.length) {
+                    return Promise.reject(new Error('RTCRtpSendParameters not found for local video track'));
+                }
+                logger.debug(`Setting max height of ${frameHeight} on local video`);
+                for (const encoding in parameters.encodings) {
+                    if (parameters.encodings.hasOwnProperty(encoding)) {
+                        parameters.encodings[encoding].active = encodingsEnabledState[encoding];
+                    }
+                }
+
+                return videoSender.setParameters(parameters);
+            });
+    }
+
+    // Apply the height constraint on the local camera track
+    const aspectRatio = (track.getSettings().width / track.getSettings().height).toPrecision(4);
+
+    logger.debug(`Setting max height of ${frameHeight} on local video`);
+
+    return track.applyConstraints(
+        {
+            aspectRatio,
+            height: {
+                ideal: frameHeight
+            }
+        });
+};
+
 /**
  * Enables/disables video media transmission on this peer connection. When
  * disabled the SDP video media direction in the local SDP will be adjusted to

modules/xmpp/JingleSessionPC.js

@@ -1328,6 +1328,16 @@ export default class JingleSessionPC extends JingleSession {
             IQ_TIMEOUT);
     }
 
+    /**
+     * Sets the resolution constraint on the local camera track.
+     * @param {number} maxFrameHeight - The user preferred max frame height.
+     * @returns {Promise} promise that will be resolved when the operation is
+     * successful and rejected otherwise.
+     */
+    setSenderVideoConstraint(maxFrameHeight) {
+        return this.peerconnection.setSenderVideoConstraint(maxFrameHeight);
+    }
+
     /**
      * @inheritDoc
      */