feat(screenSharing): Add system audio screen sharing via audio mixer

JitsiMeetJS.js

@@ -1,6 +1,7 @@
 /* global __filename */
 
 import getActiveAudioDevice from './modules/detection/ActiveDeviceDetector';
+import AudioMixer from './modules/webaudio/AudioMixer';
 import * as DetectionEvents from './modules/detection/DetectionEvents';
 import TrackVADEmitter from './modules/detection/TrackVADEmitter';
 import { createGetUserMediaEvent } from './service/statistics/AnalyticsEvents';
@@ -516,6 +517,16 @@ export default _mergeNamespaceAndModule({
         return TrackVADEmitter.create(localAudioDeviceId, sampleRate, vadProcessor);
     },
 
+    /**
+     * Create AudioMixer, which is essentially a wrapper over web audio ChannelMergerNode. It essentially allows the
+     * user to mix multiple MediaStreams into a single one.
+     *
+     * @returns {AudioMixer}
+     */
+    createAudioMixer() {
+        return new AudioMixer();
+    },
+
     /**
      * Go through all audio devices on the system and return one that is active, i.e. has audio signal.
      *

modules/RTC/JitsiLocalTrack.js

@@ -375,7 +375,9 @@ export default class JitsiLocalTrack extends JitsiTrack {
             return Promise.reject(new Error('setEffect already in progress!'));
         }
 
-        if (this.isMuted()) {
+        // In case we have an audio track that is being enhanced with an effect, we still want it to be applied,
+        // even if the track is muted. Where as for video the actual track doesn't exists if it's muted.
+        if (this.isMuted() && !this.isAudioTrack()) {
             this._streamEffect = effect;
 
             return Promise.resolve();

modules/RTC/RTCUtils.js

@@ -1287,13 +1287,32 @@ class RTCUtils extends Listenable {
 
             const { stream, sourceId, sourceType } = desktopStream;
 
-            mediaStreamsMetaData.push({
-                stream,
-                sourceId,
-                sourceType,
-                track: stream.getVideoTracks()[0],
-                videoType: VideoType.DESKTOP
-            });
+            const desktopAudioTracks = stream.getAudioTracks();
+
+            if (desktopAudioTracks.length) {
+                const desktopAudioStream = new MediaStream(desktopAudioTracks);
+
+                mediaStreamsMetaData.push({
+                    stream: desktopAudioStream,
+                    sourceId,
+                    sourceType,
+                    track: desktopAudioStream.getAudioTracks()[0]
+                });
+            }
+
+            const desktopVideoTracks = stream.getVideoTracks();
+
+            if (desktopVideoTracks.length) {
+                const desktopVideoStream = new MediaStream(desktopVideoTracks);
+
+                mediaStreamsMetaData.push({
+                    stream: desktopVideoStream,
+                    sourceId,
+                    sourceType,
+                    track: desktopVideoStream.getVideoTracks()[0],
+                    videoType: VideoType.DESKTOP
+                });
+            }
         };
 
         /**

modules/RTC/ScreenObtainer.js

@@ -312,15 +312,20 @@ const ScreenObtainer = {
             getDisplayMedia = navigator.mediaDevices.getDisplayMedia.bind(navigator.mediaDevices);
         }
 
-        getDisplayMedia({ video: true })
+        getDisplayMedia({ video: true,
+            audio: true })
             .then(stream => {
                 let applyConstraintsPromise;
 
                 if (stream
                     && stream.getTracks()
                     && stream.getTracks().length > 0) {
-                    applyConstraintsPromise = stream.getTracks()[0]
-                        .applyConstraints(options.trackOptions);
+                    const videoTrack = stream.getVideoTracks()[0];
+
+                    // Apply video track constraint.
+                    if (videoTrack) {
+                        applyConstraintsPromise = videoTrack.applyConstraints(options.trackOptions);
+                    }
                 } else {
                     applyConstraintsPromise = Promise.resolve();
                 }

modules/detection/TrackVADEmitter.js

@@ -2,7 +2,7 @@ import EventEmitter from 'events';
 
 import RTC from '../RTC/RTC';
 
-import { createAudioContext } from './webaudio/WebAudioUtils';
+import { createAudioContext } from '../webaudio/WebAudioUtils';
 import { VAD_SCORE_PUBLISHED } from './DetectionEvents';
 
 /**

modules/webaudio/AudioMixer.js

@@ -0,0 +1,91 @@
+/* global
+    __filename
+*/
+
+import { getLogger } from 'jitsi-meet-logger';
+import { createAudioContext } from './WebAudioUtils';
+
+const logger = getLogger(__filename);
+
+/**
+ * The AudioMixer, as the name implies, mixes a number of MediaStreams containing audio tracks into a single
+ * MediaStream.
+ */
+export default class AudioMixer {
+    /**
+     * Create AudioMixer instance.
+     */
+    constructor() {
+        this._started = false;
+        this._streamsToMix = [];
+    }
+
+    /**
+     * Add audio MediaStream to be mixed, if the stream doesn't contain any audio tracks it will be ignored.
+     *
+     * @param {MediaStream} stream - MediaStream to be mixed.
+     */
+    addMediaStream(stream) {
+        if (!stream.getAudioTracks()) {
+            logger.warn('Added MediaStream doesn\'t contain audio tracks.');
+        }
+
+        this._streamsToMix.push(stream);
+    }
+
+    /**
+     * At this point a WebAudio ChannelMergerNode is created and and the two associated MediaStreams are connected to
+     * it; the resulting mixed MediaStream is returned.
+     *
+     * @returns {MediaStream} - MediaStream containing added streams mixed together, or null if no MediaStream
+     * is added.
+     */
+    start() {
+        // If the mixer was already started just return the existing mixed stream.
+        if (this._started) {
+            return this._mixedMSD.stream;
+        }
+
+        this._audioContext = createAudioContext();
+
+        if (!this._streamsToMix.length) {
+            logger.warn('No MediaStream\'s added to AudioMixer, nothing will happen.');
+
+            return null;
+        }
+
+        this._started = true;
+
+        // Create ChannelMergerNode and connect all MediaStreams to it.
+        this._channelMerger = this._audioContext.createChannelMerger(this._streamsToMix.length);
+
+        for (const stream of this._streamsToMix) {
+            const streamMSS = this._audioContext.createMediaStreamSource(stream);
+
+            streamMSS.connect(this._channelMerger);
+        }
+
+        this._mixedMSD = this._audioContext.createMediaStreamDestination();
+        this._channelMerger.connect(this._mixedMSD);
+
+        return this._mixedMSD.stream;
+    }
+
+    /**
+     * Disconnect the ChannelMergerNode stopping the audio mix process.References to MediaStreams are also cleared.
+     *
+     * @returns {void}
+     */
+    reset() {
+        this._started = false;
+        this._streamsToMix = [];
+
+        if (this._channelMerger) {
+            this._channelMerger.disconnect();
+        }
+
+        if (this._audioContext) {
+            this._audioContext = undefined;
+        }
+    }
+}