ljm_j1/modules/connectivity/ConnectionQuality.js

import { getLogger } from '@jitsi/logger';

import * as ConferenceEvents from '../../JitsiConferenceEvents';
import CodecMimeType from '../../service/RTC/CodecMimeType';
import * as RTCEvents from '../../service/RTC/RTCEvents';
import * as ConnectionQualityEvents from '../../service/connectivity/ConnectionQualityEvents';
import browser from '../browser';

const Resolutions = require('../../service/RTC/Resolutions');
const VideoType = require('../../service/RTC/VideoType');
const { XMPPEvents } = require('../../service/xmpp/XMPPEvents');

const logger = getLogger(__filename);

/**
 * The value to use for the "type" field for messages sent by ConnectionQuality
 * over the data channel.
 */
const STATS_MESSAGE_TYPE = 'stats';

/**
 * The value to use for the "type" field for messages sent
 * over the data channel that contain facial expression.
 */
const FACIAL_EXPRESSION_MESSAGE_TYPE = 'facial_expression';

const kSimulcastFormats = [
    { width: 1920,
        height: 1080,
        layers: 3,
        target: 'high',
        targetRN: 4000000 },
    { width: 1280,
        height: 720,
        layers: 3,
        target: 'high',
        targetRN: 2500000 },
    { width: 960,
        height: 540,
        layers: 3,
        target: 'standard',
        targetRN: 900000 },
    { width: 640,
        height: 360,
        layers: 2,
        target: 'standard',
        targetRN: 500000 },
    { width: 480,
        height: 270,
        layers: 2,
        target: 'low',
        targetRN: 350000 },
    { width: 320,
        height: 180,
        layers: 1,
        target: 'low',
        targetRN: 150000 }
];

/**
 * The maximum bitrate to use as a measurement against the participant's current
 * bitrate. This cap helps in the cases where the participant's bitrate is high
 * but not enough to fulfill high targets, such as with 1080p.
 */
const MAX_TARGET_BITRATE = 2500;

/**
 * The initial bitrate for video in kbps.
 */
let startBitrate = 800;

/**
 * Gets the expected bitrate (in kbps) in perfect network conditions.
 * @param simulcast {boolean} whether simulcast is enabled or not.
 * @param resolution {Resolution} the resolution.
 * @param millisSinceStart {number} the number of milliseconds since sending video started.
 * @param videoQualitySettings {Object} the bitrate and codec settings for the local video source.
 */
function getTarget(simulcast, resolution, millisSinceStart, videoQualitySettings) {
    let target = 0;
    let height = Math.min(resolution.height, resolution.width);

    // Find the first format with height no bigger than ours.
    let simulcastFormat = kSimulcastFormats.find(f => f.height <= height);

    if (simulcastFormat && simulcast && videoQualitySettings.codec === CodecMimeType.VP8) {
        // Sum the target fields from all simulcast layers for the given
        // resolution (e.g. 720p + 360p + 180p) for VP8 simulcast.
        for (height = simulcastFormat.height; height >= 180; height /= 2) {
            const targetHeight = height;

            simulcastFormat = kSimulcastFormats.find(f => f.height === targetHeight);
            if (simulcastFormat) {
                target += browser.isReactNative()
                    ? simulcastFormat.targetRN
                    : videoQualitySettings[simulcastFormat.target];
            } else {
                break;
            }
        }
    } else if (simulcastFormat) {
        // For VP9 SVC, H.264 (simulcast automatically disabled) and p2p, target bitrate will be
        // same as that of the individual stream bitrate.
        target = browser.isReactNative()
            ? simulcastFormat.targetRN
            : videoQualitySettings[simulcastFormat.target];
    }

    // Allow for an additional 1 second for ramp up -- delay any initial drop
    // of connection quality by 1 second. Convert target from bps to kbps.
    return Math.min(target / 1000, rampUp(Math.max(0, millisSinceStart - 1000)));
}

/**
 * Gets the bitrate to which GCC would have ramped up in perfect network
 * conditions after millisSinceStart milliseconds.
 * @param millisSinceStart {number} the number of milliseconds since sending
 * video was enabled.
 */
function rampUp(millisSinceStart) {
    if (millisSinceStart > 60000) {
        return Number.MAX_SAFE_INTEGER;
    }

    // According to GCC the send side bandwidth estimation grows with at most
    // 8% per second.
    // https://tools.ietf.org/html/draft-ietf-rmcat-gcc-02#section-5.5
    return startBitrate * Math.pow(1.08, millisSinceStart / 1000);
}

/**
 * A class which monitors the local statistics coming from the RTC modules, and
 * calculates a "connection quality" value, in percent, for the media
 * connection. A value of 100% indicates a very good network connection, and a
 * value of 0% indicates a poor connection.
 */
export default class ConnectionQuality {
    /**
     *
     * @param conference
     * @param eventEmitter
     * @param options
     */
    constructor(conference, eventEmitter, options) {
        this.eventEmitter = eventEmitter;

        /**
         * The owning JitsiConference.
         */
        this._conference = conference;

        /**
         * Holds statistics about the local connection quality.
         */
        this._localStats = {
            connectionQuality: 100,
            jvbRTT: undefined
        };

        /**
         * The time this._localStats.connectionQuality was last updated.
         */
        this._lastConnectionQualityUpdate = -1;

        /**
         * Conference options.
         */
        this._options = options;

        /**
         * Maps a participant ID to an object holding connection quality
         * statistics received from this participant.
         */
        this._remoteStats = {};

        /**
         * The time that the ICE state last changed to CONNECTED. We use this
         * to calculate how much time we as a sender have had to ramp-up.
         */
        this._timeIceConnected = -1;

        /**
         * The time that local video was unmuted. We use this to calculate how
         * much time we as a sender have had to ramp-up.
         */
        this._timeVideoUnmuted = -1;

        // We assume a global startBitrate value for the sake of simplicity.
        if (this._options.config?.startBitrate > 0) {
            startBitrate = this._options.config.startBitrate;
        }

        // TODO: consider ignoring these events and letting the user of
        // lib-jitsi-meet handle these separately.
        conference.on(
            ConferenceEvents.CONNECTION_INTERRUPTED,
            () => {
                this._updateLocalConnectionQuality(0);
                this.eventEmitter.emit(
                    ConnectionQualityEvents.LOCAL_STATS_UPDATED,
                    this._localStats);
                this._broadcastLocalStats();
            });

        conference.room.addListener(
            XMPPEvents.ICE_CONNECTION_STATE_CHANGED,
            (jingleSession, newState) => {
                if (!jingleSession.isP2P && newState === 'connected') {
                    this._timeIceConnected = window.performance.now();
                }
            });

        // Listen to DataChannel message from other participants in the
        // conference, and update the _remoteStats field accordingly.
        // TODO - Delete this when all the mobile endpoints switch to using the new Colibri
        // message format for sending the endpoint stats.
        conference.on(
            ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
            (participant, payload) => {
                if (payload.type === STATS_MESSAGE_TYPE) {
                    this._updateRemoteStats(
                        participant.getId(), payload.values);
                }
            });

        conference.on(
            ConferenceEvents.ENDPOINT_STATS_RECEIVED,
            (participant, payload) => {
                this._updateRemoteStats(participant.getId(), payload);
            });

        conference.on(
            ConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
            (participant, payload) => {
                if (payload.type === FACIAL_EXPRESSION_MESSAGE_TYPE) {
                    this.eventEmitter.emit(
                        ConferenceEvents.FACIAL_EXPRESSION_ADDED,
                        participant.getId(),
                        payload);
                }
            });

        // Listen to local statistics events originating from the RTC module and update the _localStats field.
        conference.statistics.addConnectionStatsListener(this._updateLocalStats.bind(this));

        // Save the last time we were unmuted.
        conference.on(
            ConferenceEvents.TRACK_MUTE_CHANGED,
            track => {
                if (track.isVideoTrack()) {
                    if (track.isMuted()) {
                        this._timeVideoUnmuted = -1;
                    } else {
                        this._maybeUpdateUnmuteTime();
                    }
                }
            });
        conference.on(
            ConferenceEvents.TRACK_ADDED,
            track => {
                if (track.isVideoTrack() && !track.isMuted()) {
                    this._maybeUpdateUnmuteTime();
                }
            });
        conference.rtc.on(
            RTCEvents.LOCAL_TRACK_MAX_ENABLED_RESOLUTION_CHANGED,
            track => {
                this._localStats.maxEnabledResolution = track.maxEnabledResolution;
            });

        conference.on(
            ConferenceEvents.SERVER_REGION_CHANGED,
            serverRegion => {
                this._localStats.serverRegion = serverRegion;
            });

        conference.on(
            ConferenceEvents.PROPERTIES_CHANGED,
            properties => {
                this._localStats.bridgeCount
                    = Number((properties || {})['bridge-count']);
            }
        );
    }

    /**
     * Sets _timeVideoUnmuted if it was previously unset. If it was already set,
     * doesn't change it.
     */
    _maybeUpdateUnmuteTime() {
        if (this._timeVideoUnmuted < 0) {
            this._timeVideoUnmuted = window.performance.now();
        }
    }

    /**
     * Calculates a new "connection quality" value.
     * @param videoType {VideoType} the type of the video source (camera or a screen capture).
     * @param isMuted {boolean} whether the local video is muted.
     * @param resolutionName {Resolution} the input resolution used by the camera.
     * @returns {*} the newly calculated connection quality.
     */
    _calculateConnectionQuality(videoType, isMuted, resolutionName) {

        // resolutionName is an index into Resolutions (where "720" is
        // "1280x720" and "960" is "960x720" ...).
        const resolution = Resolutions[resolutionName];

        let quality = 100;
        let packetLoss;

        // TODO: take into account packet loss for received streams

        if (this._localStats.packetLoss) {
            packetLoss = this._localStats.packetLoss.upload;

            // Ugly Hack Alert (UHA):
            // The packet loss for the upload direction is calculated based on
            // incoming RTCP Receiver Reports. Since we don't have RTCP
            // termination for audio, these reports come from the actual
            // receivers in the conference and therefore the reported packet
            // loss includes loss from the bridge to the receiver.
            // When we are sending video this effect is small, because the
            // number of video packets is much larger than the number of audio
            // packets (and our calculation is based on the total number of
            // received and lost packets).
            // When video is muted, however, the effect might be significant,
            // but we don't know what it is. We do know that it is positive, so
            // as a temporary solution, until RTCP termination is implemented
            // for the audio streams, we relax the packet loss checks here.
            if (isMuted) {
                packetLoss *= 0.5;
            }
        }

        if (isMuted || !resolution || videoType === VideoType.DESKTOP
            || this._timeIceConnected < 0
            || this._timeVideoUnmuted < 0) {

            // Calculate a value based on packet loss only.
            if (packetLoss === undefined) {
                logger.error('Cannot calculate connection quality, unknown '
                    + 'packet loss.');
                quality = 100;
            } else if (packetLoss <= 2) {
                quality = 100; // Full 5 bars.
            } else if (packetLoss <= 4) {
                quality = 70; // 4 bars
            } else if (packetLoss <= 6) {
                quality = 50; // 3 bars
            } else if (packetLoss <= 8) {
                quality = 30; // 2 bars
            } else if (packetLoss <= 12) {
                quality = 10; // 1 bars
            } else {
                quality = 0; // Still 1 bar, but slower climb-up.
            }
        } else {
            // Calculate a value based on the send video bitrate on the active TPC.
            const activeTPC = this._conference.getActivePeerConnection();

            if (activeTPC) {
                const isSimulcastOn = activeTPC.isSimulcastOn();
                const videoQualitySettings = activeTPC.getTargetVideoBitrates();

                // Add the codec info as well.
                videoQualitySettings.codec = activeTPC.getConfiguredVideoCodec();

                // Time since sending of video was enabled.
                const millisSinceStart = window.performance.now()
                    - Math.max(this._timeVideoUnmuted, this._timeIceConnected);
                const statsInterval = this._options.config?.pcStatsInterval ?? 10000;

                // Expected sending bitrate in perfect conditions.
                let target = getTarget(isSimulcastOn, resolution, millisSinceStart, videoQualitySettings);

                target = Math.min(target, MAX_TARGET_BITRATE);

                // Calculate the quality only after the stats are available (after video was enabled).
                if (millisSinceStart > statsInterval) {
                    quality = 100 * this._localStats.bitrate.upload / target;
                }
            }

            // Whatever the bitrate, drop early if there is significant loss
            if (packetLoss && packetLoss >= 10) {
                quality = Math.min(quality, 30);
            }
        }

        // Make sure that the quality doesn't climb quickly
        if (this._lastConnectionQualityUpdate > 0) {
            const maxIncreasePerSecond = 2;
            const prevConnectionQuality = this._localStats.connectionQuality;
            const diffSeconds = (window.performance.now() - this._lastConnectionQualityUpdate) / 1000;

            quality = Math.min(quality, prevConnectionQuality + (diffSeconds * maxIncreasePerSecond));
        }

        return Math.min(100, quality);
    }

    /**
     * Updates the localConnectionQuality value
     * @param values {number} the new value. Should be in [0, 100].
     */
    _updateLocalConnectionQuality(value) {
        this._localStats.connectionQuality = value;
        this._lastConnectionQualityUpdate = window.performance.now();
    }

    /**
     * Broadcasts the local statistics to all other participants in the
     * conference.
     */
    _broadcastLocalStats() {
        // Send only the data that remote participants care about.
        const data = {
            bitrate: this._localStats.bitrate,
            packetLoss: this._localStats.packetLoss,
            connectionQuality: this._localStats.connectionQuality,
            jvbRTT: this._localStats.jvbRTT,
            serverRegion: this._localStats.serverRegion,
            maxEnabledResolution: this._localStats.maxEnabledResolution,
            avgAudioLevels: this._localStats.localAvgAudioLevels
        };

        try {
            this._conference.sendEndpointStatsMessage(data);
        } catch (err) {
            // Ignore the error as we might hit it in the beginning of the call before the channel is ready.
            // The statistics will be sent again after few seconds and error is logged elseware as well.
        }
    }

    /**
     * Updates the local statistics
     * @param {TraceablePeerConnection} tpc the peerconnection which emitted
     * the stats
     * @param data new statistics
     */
    _updateLocalStats(tpc, data) {
        // Update jvbRTT
        if (!tpc.isP2P) {
            const jvbRTT
                = data.transport
                    && data.transport.length && data.transport[0].rtt;

            this._localStats.jvbRTT = jvbRTT ? jvbRTT : undefined;
        }

        // Do not continue with processing of other stats if they do not
        // originate from the active peerconnection
        if (tpc !== this._conference.getActivePeerConnection()) {
            return;
        }

        let key;
        const updateLocalConnectionQuality
            = !this._conference.isConnectionInterrupted();
        const localVideoTrack
            = this._conference.getLocalVideoTrack();
        const videoType
            = localVideoTrack ? localVideoTrack.videoType : undefined;
        const isMuted = localVideoTrack ? localVideoTrack.isMuted() : true;
        const resolution = localVideoTrack
            ? Math.min(localVideoTrack.resolution, localVideoTrack.maxEnabledResolution) : null;

        if (!isMuted) {
            this._maybeUpdateUnmuteTime();
        }

        // Copy the fields already in 'data'.
        for (key in data) {
            if (data.hasOwnProperty(key)) {
                this._localStats[key] = data[key];
            }
        }

        // And re-calculate the connectionQuality field.
        if (updateLocalConnectionQuality) {
            this._updateLocalConnectionQuality(
                this._calculateConnectionQuality(
                    videoType,
                    isMuted,
                    resolution));
        }

        this.eventEmitter.emit(
            ConnectionQualityEvents.LOCAL_STATS_UPDATED,
            this._localStats);
        this._broadcastLocalStats();
    }

    /**
     * Updates remote statistics
     * @param id the id of the remote participant
     * @param data the statistics received
     */
    _updateRemoteStats(id, data) {
        // Use only the fields we need
        this._remoteStats[id] = {
            bitrate: data.bitrate,
            packetLoss: data.packetLoss,
            connectionQuality: data.connectionQuality,
            jvbRTT: data.jvbRTT,
            serverRegion: data.serverRegion,
            maxEnabledResolution: data.maxEnabledResolution,
            avgAudioLevels: data.avgAudioLevels
        };

        this.eventEmitter.emit(
            ConnectionQualityEvents.REMOTE_STATS_UPDATED,
            id,
            this._remoteStats[id]);
    }

    /**
     * Returns the local statistics.
     * Exported only for use in jitsi-meet-torture.
     */
    getStats() {
        return this._localStats;
    }
}