lib-jitsi-meet/modules/RTC/JitsiLocalTrack.js

/* global __filename, Promise */

import CameraFacingMode from '../../service/RTC/CameraFacingMode';
import { getLogger } from 'jitsi-meet-logger';
import JitsiTrack from './JitsiTrack';
import JitsiTrackError from '../../JitsiTrackError';
import * as JitsiTrackErrors from '../../JitsiTrackErrors';
import * as JitsiTrackEvents from '../../JitsiTrackEvents';
import * as MediaType from '../../service/RTC/MediaType';
import RTCBrowserType from './RTCBrowserType';
import RTCEvents from '../../service/RTC/RTCEvents';
import RTCUtils from './RTCUtils';
import Statistics from '../statistics/statistics';
import VideoType from '../../service/RTC/VideoType';

const logger = getLogger(__filename);

/**
 * Creates Promise for mute/unmute operation.
 *
 * @param {JitsiLocalTrack} track - The track that will be muted/unmuted.
 * @param {boolean} mute - Whether to mute or unmute the track.
 * @returns {Promise}
 */
function createMuteUnmutePromise(track, mute) {
    if (track.inMuteOrUnmuteProgress) {
        return Promise.reject(
            new JitsiTrackError(JitsiTrackErrors.TRACK_MUTE_UNMUTE_IN_PROGRESS)
        );
    }

    track.inMuteOrUnmuteProgress = true;

    return track._setMute(mute)
        .then(() => {
            track.inMuteOrUnmuteProgress = false;
        })
        .catch(status => {
            track.inMuteOrUnmuteProgress = false;
            throw status;
        });
}

/**
 * Represents a single media track(either audio or video).
 * One <tt>JitsiLocalTrack</tt> corresponds to one WebRTC MediaStreamTrack.
 */
export default class JitsiLocalTrack extends JitsiTrack {
    /**
     * Constructs new JitsiLocalTrack instanse.
     * @param {Object} trackInfo
     * @param {number} trackInfo.rtcId the ID assigned by the RTC module
     * @param trackInfo.stream WebRTC MediaStream, parent of the track
     * @param trackInfo.track underlying WebRTC MediaStreamTrack for new
     * JitsiRemoteTrack
     * @param trackInfo.mediaType the MediaType of the JitsiRemoteTrack
     * @param trackInfo.videoType the VideoType of the JitsiRemoteTrack
     * @param trackInfo.resolution the video resolution if it's a video track
     * @param trackInfo.deviceId the ID of the local device for this track
     * @param trackInfo.facingMode the camera facing mode used in getUserMedia
     * call
     * @param {sourceId} trackInfo.sourceId - The id of the desktop sharing
     * source. NOTE: defined for desktop sharing tracks only.
     * @constructor
     */
    constructor(trackInfo) {
        const {
            rtcId,
            stream,
            track,
            mediaType,
            videoType,
            resolution,
            deviceId,
            facingMode,
            sourceId
        } = trackInfo;

        super(
            null /* RTC */,
            stream,
            track,
            () => {
                if (!this.dontFireRemoveEvent) {
                    this.emit(JitsiTrackEvents.LOCAL_TRACK_STOPPED);
                }
                this.dontFireRemoveEvent = false;
            } /* inactiveHandler */,
            mediaType,
            videoType);

        /**
         * The ID assigned by the RTC module on instance creation.
         * @type {number}
         */
        this.rtcId = rtcId;
        this.dontFireRemoveEvent = false;
        this.resolution = resolution;
        this.sourceId = sourceId;

        // FIXME: currently firefox is ignoring our constraints about
        // resolutions so we do not store it, to avoid wrong reporting of local
        // track resolution
        if (RTCBrowserType.isFirefox()) {
            this.resolution = null;
        }

        this.deviceId = deviceId;
        this.storedMSID = this.getMSID();
        this.inMuteOrUnmuteProgress = false;

        /**
         * The facing mode of the camera from which this JitsiLocalTrack
         * instance was obtained.
         *
         * @private
         * @type {CameraFacingMode|undefined}
         */
        this._facingMode = facingMode;

        // Currently there is no way to know the MediaStreamTrack ended due to
        // to device disconnect in Firefox through e.g. "readyState" property.
        // Instead we will compare current track's label with device labels from
        // enumerateDevices() list.
        this._trackEnded = false;

        /**
         * The value of bytes sent received from the statistics module.
         */
        this._bytesSent = null;

        /**
         * Used only for detection of audio problems. We want to check only once
         * whether the track is sending bytes ot not. This flag is set to false
         * after the check.
         */
        this._testByteSent = true;

        // Currently there is no way to determine with what device track was
        // created (until getConstraints() support), however we can associate
        // tracks with real devices obtained from enumerateDevices() call as
        // soon as it's called.
        this._realDeviceId = this.deviceId === '' ? undefined : this.deviceId;

        /**
         * Indicates that we have called RTCUtils.stopMediaStream for the
         * MediaStream related to this JitsiTrack object.
         */
        this.stopStreamInProgress = false;

        /**
         * On mute event we are waiting for 3s to check if the stream is going
         * to be still muted before firing the event for camera issue detected
         * (NO_DATA_FROM_SOURCE).
         */
        this._noDataFromSourceTimeout = null;

        this._onDeviceListChanged = devices => {
            this._setRealDeviceIdFromDeviceList(devices);

            // Mark track as ended for those browsers that do not support
            // "readyState" property. We do not touch tracks created with
            // default device ID "".
            if (typeof this.getTrack().readyState === 'undefined'
                    && typeof this._realDeviceId !== 'undefined'
                    && !devices.find(d => d.deviceId === this._realDeviceId)) {
                this._trackEnded = true;
            }
        };

        // Subscribe each created local audio track to
        // RTCEvents.AUDIO_OUTPUT_DEVICE_CHANGED event. This is different from
        // handling this event for remote tracks (which are handled in RTC.js),
        // because there might be local tracks not attached to a conference.
        if (this.isAudioTrack() && RTCUtils.isDeviceChangeAvailable('output')) {
            this._onAudioOutputDeviceChanged = this.setAudioOutput.bind(this);
            RTCUtils.addListener(
                RTCEvents.AUDIO_OUTPUT_DEVICE_CHANGED,
                this._onAudioOutputDeviceChanged);
        }

        RTCUtils.addListener(
            RTCEvents.DEVICE_LIST_CHANGED,
            this._onDeviceListChanged);

        this._initNoDataFromSourceHandlers();
    }

    /**
     * Returns if associated MediaStreamTrack is in the 'ended' state
     * @returns {boolean}
     */
    isEnded() {
        return this.getTrack().readyState === 'ended' || this._trackEnded;
    }

    /**
     * Sets handlers to the MediaStreamTrack object that will detect camera
     * issues.
     */
    _initNoDataFromSourceHandlers() {
        if (this.isVideoTrack() && this.videoType === VideoType.CAMERA) {
            const _onNoDataFromSourceError
                = this._onNoDataFromSourceError.bind(this);

            this._setHandler('track_mute', () => {
                if (this._checkForCameraIssues()) {
                    const now = window.performance.now();

                    this._noDataFromSourceTimeout
                        = setTimeout(_onNoDataFromSourceError, 3000);
                    this._setHandler('track_unmute', () => {
                        this._clearNoDataFromSourceMuteResources();
                        Statistics.sendEventToAll(
                            `${this.getType()}.track_unmute`,
                            { value: window.performance.now() - now });
                    });
                }
            });
            this._setHandler('track_ended', _onNoDataFromSourceError);
        }
    }

    /**
     * Clears all timeouts and handlers set on MediaStreamTrack mute event.
     * FIXME: Change the name of the method with better one.
     */
    _clearNoDataFromSourceMuteResources() {
        if (this._noDataFromSourceTimeout) {
            clearTimeout(this._noDataFromSourceTimeout);
            this._noDataFromSourceTimeout = null;
        }
        this._setHandler('track_unmute', undefined);
    }

    /**
     * Called when potential camera issue is detected. Clears the handlers and
     * timeouts set on MediaStreamTrack muted event. Verifies that the camera
     * issue persists and fires NO_DATA_FROM_SOURCE event.
     */
    _onNoDataFromSourceError() {
        this._clearNoDataFromSourceMuteResources();
        if (this._checkForCameraIssues()) {
            this._fireNoDataFromSourceEvent();
        }
    }

    /**
     * Fires JitsiTrackEvents.NO_DATA_FROM_SOURCE and logs it to analytics and
     * callstats.
     */
    _fireNoDataFromSourceEvent() {
        this.emit(JitsiTrackEvents.NO_DATA_FROM_SOURCE);
        const eventName = `${this.getType()}.no_data_from_source`;

        Statistics.analytics.sendEvent(eventName);
        const log = { name: eventName };

        if (this.isAudioTrack()) {
            log.isReceivingData = this._isReceivingData();
        }
        Statistics.sendLog(JSON.stringify(log));
    }

    /**
     * Sets real device ID by comparing track information with device
     * information. This is temporary solution until getConstraints() method
     * will be implemented in browsers.
     * @param {MediaDeviceInfo[]} devices - list of devices obtained from
     *  enumerateDevices() call
     */
    _setRealDeviceIdFromDeviceList(devices) {
        const track = this.getTrack();

        // FIXME for temasys video track, label refers to id not the actual
        // device
        const device = devices.find(
            d => d.kind === `${track.kind}input` && d.label === track.label);

        if (device) {
            this._realDeviceId = device.deviceId;
        }
    }

    /**
     * Sets the stream property of JitsiLocalTrack object and sets all stored
     * handlers to it.
     * @param {MediaStream} stream the new stream.
     */
    _setStream(stream) {
        super._setStream(stream);

        // Store the MSID for video mute/unmute purposes
        if (stream) {
            this.storedMSID = this.getMSID();
            logger.debug(`Setting new MSID: ${this.storedMSID} on ${this}`);
        } else {
            logger.debug(`Setting 'null' stream on ${this}`);
        }
    }

    /**
     * Mutes the track. Will reject the Promise if there is mute/unmute
     * operation in progress.
     * @returns {Promise}
     */
    mute() {
        return createMuteUnmutePromise(this, true);
    }

    /**
     * Unmutes the track. Will reject the Promise if there is mute/unmute
     * operation in progress.
     * @returns {Promise}
     */
    unmute() {
        return createMuteUnmutePromise(this, false);
    }

    /**
     * Mutes / unmutes the track.
     *
     * @param {boolean} mute - If true the track will be muted. Otherwise the
     * track will be unmuted.
     * @private
     * @returns {Promise}
     */
    _setMute(mute) {
        if (this.isMuted() === mute) {
            return Promise.resolve();
        }

        let promise = Promise.resolve();

        this.dontFireRemoveEvent = false;

        // A function that will print info about muted status transition
        const logMuteInfo = () => logger.info(`Mute ${this}: ${mute}`);

        if (this.isAudioTrack()
            || this.videoType === VideoType.DESKTOP
            || !RTCBrowserType.doesVideoMuteByStreamRemove()) {
            logMuteInfo();
            if (this.track) {
                this.track.enabled = !mute;
            }
        } else if (mute) {
            this.dontFireRemoveEvent = true;
            promise = new Promise((resolve, reject) => {
                logMuteInfo();
                this._removeStreamFromConferenceAsMute(() => {
                    // FIXME: Maybe here we should set the SRC for the
                    // containers to something
                    this._stopMediaStream();
                    this._setStream(null);
                    resolve();
                }, err => {
                    reject(err);
                });
            });
        } else {
            logMuteInfo();

            // This path is only for camera.
            const streamOptions = {
                cameraDeviceId: this.getDeviceId(),
                devices: [ MediaType.VIDEO ],
                facingMode: this.getCameraFacingMode()
            };

            if (this.resolution) {
                streamOptions.resolution = this.resolution;
            }

            promise = RTCUtils.obtainAudioAndVideoPermissions(streamOptions)
                .then(streamsInfo => {
                    const mediaType = this.getType();
                    const streamInfo = streamsInfo.find(
                            info => info.mediaType === mediaType);

                    if (streamInfo) {
                        this._setStream(streamInfo.stream);
                        this.track = streamInfo.track;

                        // This is not good when video type changes after
                        // unmute, but let's not crash here
                        if (this.videoType !== streamInfo.videoType) {
                            logger.warn(
                                `${this}: video type has changed after unmute!`,
                                this.videoType, streamInfo.videoType);
                            this.videoType = streamInfo.videoType;
                        }
                    } else {
                        throw new JitsiTrackError(
                            JitsiTrackErrors.TRACK_NO_STREAM_FOUND);
                    }

                    this.containers = this.containers.map(
                        cont => RTCUtils.attachMediaStream(cont, this.stream));

                    return this._addStreamToConferenceAsUnmute();
                });
        }

        return promise
            .then(() => this._sendMuteStatus(mute))
            .then(() => {
                this.emit(JitsiTrackEvents.TRACK_MUTE_CHANGED, this);
            });
    }

    /**
     * Adds stream to conference and marks it as "unmute" operation.
     *
     * @private
     * @returns {Promise}
     */
    _addStreamToConferenceAsUnmute() {
        if (!this.conference) {
            return Promise.resolve();
        }

        // FIXME it would be good to not included conference as part of this
        // process. Only TraceablePeerConnections to which the track is attached
        // should care about this action. The TPCs to which the track is not
        // attached can sync up when track is re-attached.
        // A problem with that is that the "modify sources" queue is part of
        // the JingleSessionPC and it would be excluded from the process. One
        // solution would be to extract class between TPC and JingleSessionPC
        // which would contain the queue and would notify the signaling layer
        // when local SSRCs are changed. This would help to separate XMPP from
        // the RTC module.
        return new Promise((resolve, reject) => {
            this.conference._addLocalTrackAsUnmute(this)
                .then(resolve, error => reject(new Error(error)));
        });
    }

    /**
     * Removes stream from conference and marks it as "mute" operation.
     * @param {Function} successCallback will be called on success
     * @param {Function} errorCallback will be called on error
     * @private
     */
    _removeStreamFromConferenceAsMute(successCallback, errorCallback) {
        if (!this.conference) {
            successCallback();

            return;
        }
        this.conference._removeLocalTrackAsMute(this).then(
            successCallback,
            error => errorCallback(new Error(error)));
    }

    /**
     * Sends mute status for a track to conference if any.
     *
     * @param {boolean} mute - If track is muted.
     * @private
     * @returns {Promise}
     */
    _sendMuteStatus(mute) {
        if (!this.conference || !this.conference.room) {
            return Promise.resolve();
        }

        return new Promise(resolve => {
            this.conference.room[
                this.isAudioTrack()
                    ? 'setAudioMute'
                    : 'setVideoMute'](mute, resolve);
        });
    }

    /**
     * @inheritdoc
     *
     * Stops sending the media track. And removes it from the HTML.
     * NOTE: Works for local tracks only.
     *
     * @extends JitsiTrack#dispose
     * @returns {Promise}
     */
    dispose() {
        let promise = Promise.resolve();

        if (this.conference) {
            promise = this.conference.removeTrack(this);
        }

        if (this.stream) {
            this._stopMediaStream();
            this.detach();
        }

        RTCUtils.removeListener(RTCEvents.DEVICE_LIST_CHANGED,
            this._onDeviceListChanged);

        if (this._onAudioOutputDeviceChanged) {
            RTCUtils.removeListener(RTCEvents.AUDIO_OUTPUT_DEVICE_CHANGED,
                this._onAudioOutputDeviceChanged);
        }

        return promise.then(() => super.dispose());
    }

    /**
     * Returns <tt>true</tt> - if the stream is muted
     * and <tt>false</tt> otherwise.
     * @returns {boolean} <tt>true</tt> - if the stream is muted
     * and <tt>false</tt> otherwise.
     */
    isMuted() {
        // this.stream will be null when we mute local video on Chrome
        if (!this.stream) {
            return true;
        }
        if (this.isVideoTrack() && !this.isActive()) {
            return true;
        }

        return !this.track || !this.track.enabled;

    }

    /**
     * Sets the JitsiConference object associated with the track. This is temp
     * solution.
     * @param conference the JitsiConference object
     */
    _setConference(conference) {
        this.conference = conference;

        // We want to keep up with postponed events which should have been fired
        // on "attach" call, but for local track we not always have the
        // conference before attaching. However this may result in duplicated
        // events if they have been triggered on "attach" already.
        for (let i = 0; i < this.containers.length; i++) {
            this._maybeFireTrackAttached(this.containers[i]);
        }
    }

    /**
     * Returns <tt>true</tt>.
     * @returns {boolean} <tt>true</tt>
     */
    isLocal() {
        return true;
    }

    /**
     * Returns device id associated with track.
     * @returns {string}
     */
    getDeviceId() {
        return this._realDeviceId || this.deviceId;
    }

    /**
     * Returns the participant id which owns the track.
     * @returns {string} the id of the participants. It corresponds to the
     * Colibri endpoint id/MUC nickname in case of Jitsi-meet.
     */
    getParticipantId() {
        return this.conference && this.conference.myUserId();
    }

    /**
     * Sets the value of bytes sent statistic.
     * @param {TraceablePeerConnection} tpc the source of the "bytes sent" stat
     * @param {number} bytesSent the new value
     * NOTE: used only for audio tracks to detect audio issues.
     */
    _setByteSent(tpc, bytesSent) {
        this._bytesSent = bytesSent;
        const iceConnectionState = tpc.getConnectionState();

        if (this._testByteSent && iceConnectionState === 'connected') {
            setTimeout(() => {
                if (this._bytesSent <= 0) {
                    // FIXME: Remove ${this}
                    logger.warn(`${this} 'bytes sent' <= 0: \
                        ${this._bytesSent}`);

                    // we are not receiving anything from the microphone
                    this._fireNoDataFromSourceEvent();
                }
            }, 3000);
            this._testByteSent = false;
        }
    }

    /**
     * Returns facing mode for video track from camera. For other cases (e.g.
     * audio track or 'desktop' video track) returns undefined.
     *
     * @returns {CameraFacingMode|undefined}
     */
    getCameraFacingMode() {
        if (this.isVideoTrack() && this.videoType === VideoType.CAMERA) {
            // MediaStreamTrack#getSettings() is not implemented in many
            // browsers, so we need feature checking here. Progress on the
            // respective browser's implementation can be tracked at
            // https://bugs.chromium.org/p/webrtc/issues/detail?id=2481 for
            // Chromium and https://bugzilla.mozilla.org/show_bug.cgi?id=1213517
            // for Firefox. Even if a browser implements getSettings() already,
            // it might still not return anything for 'facingMode'.
            let trackSettings;

            try {
                trackSettings = this.track.getSettings();
            } catch (e) {
                // XXX React-native-webrtc, for example, defines
                // MediaStreamTrack#getSettings() but the implementation throws
                // a "Not implemented" Error.
            }
            if (trackSettings && 'facingMode' in trackSettings) {
                return trackSettings.facingMode;
            }

            if (typeof this._facingMode !== 'undefined') {
                return this._facingMode;
            }

            // In most cases we are showing a webcam. So if we've gotten here,
            // it should be relatively safe to assume that we are probably
            // showing the user-facing camera.
            return CameraFacingMode.USER;
        }

        return undefined;
    }

    /**
     * Stops the associated MediaStream.
     */
    _stopMediaStream() {
        this.stopStreamInProgress = true;
        RTCUtils.stopMediaStream(this.stream);
        this.stopStreamInProgress = false;
    }

    /**
     * Switches the camera facing mode if the WebRTC implementation supports the
     * custom MediaStreamTrack._switchCamera method. Currently, the method in
     * question is implemented in react-native-webrtc only. When such a WebRTC
     * implementation is executing, the method is the preferred way to switch
     * between the front/user-facing and the back/environment-facing cameras
     * because it will likely be (as is the case of react-native-webrtc)
     * noticeably faster that creating a new MediaStreamTrack via a new
     * getUserMedia call with the switched facingMode constraint value.
     * Moreover, the approach with a new getUserMedia call may not even work:
     * WebRTC on Android and iOS is either very slow to open the camera a second
     * time or plainly freezes attempting to do that.
     */
    _switchCamera() {
        if (this.isVideoTrack()
                && this.videoType === VideoType.CAMERA
                && typeof this.track._switchCamera === 'function') {
            this.track._switchCamera();

            this._facingMode
                = this._facingMode === CameraFacingMode.ENVIRONMENT
                    ? CameraFacingMode.USER
                    : CameraFacingMode.ENVIRONMENT;
        }
    }

    /**
     * Detects camera issues on ended and mute events from MediaStreamTrack.
     * @returns {boolean} true if an issue is detected and false otherwise
     */
    _checkForCameraIssues() {
        if (!this.isVideoTrack() || this.stopStreamInProgress
            || this.videoType === VideoType.DESKTOP) {
            return false;
        }

        return !this._isReceivingData();
    }

    /**
     * Checks whether the attached MediaStream is receiving data from source or
     * not. If the stream property is null(because of mute or another reason)
     * this method will return false.
     * NOTE: This method doesn't indicate problem with the streams directly.
     * For example in case of video mute the method will return false or if the
     * user has disposed the track.
     * @returns {boolean} true if the stream is receiving data and false
     * this otherwise.
     */
    _isReceivingData() {
        if (!this.stream) {
            return false;
        }

        // In older version of the spec there is no muted property and
        // readyState can have value muted. In the latest versions
        // readyState can have values "live" and "ended" and there is
        // muted boolean property. If the stream is muted that means that
        // we aren't receiving any data from the source. We want to notify
        // the users for error if the stream is muted or ended on it's
        // creation.
        return this.stream.getTracks().some(track =>
            (!('readyState' in track) || track.readyState === 'live')
                && (!('muted' in track) || track.muted !== true));
    }

    /**
     * Creates a text representation of this local track instance.
     * @return {string}
     */
    toString() {
        return `LocalTrack[${this.rtcId},${this.getType()}]`;
    }

}