feat(screenshare): Audio only screenshare (#8922)

* audio only screen share implementation

* clean up

* handle stop screen share from chrome window

* update icon

conference.js

@@ -6,6 +6,7 @@ import Logger from 'jitsi-meet-logger';
 
 import { openConnection } from './connection';
 import { ENDPOINT_TEXT_MESSAGE_NAME } from './modules/API/constants';
+import { AUDIO_ONLY_SCREEN_SHARE_NO_TRACK } from './modules/UI/UIErrors';
 import AuthHandler from './modules/UI/authentication/AuthHandler';
 import UIUtil from './modules/UI/util/UIUtil';
 import mediaDeviceHelper from './modules/devices/mediaDeviceHelper';
@@ -126,6 +127,7 @@ import {
     makePrecallTest
 } from './react/features/prejoin';
 import { disableReceiver, stopReceiver } from './react/features/remote-control';
+import { setScreenAudioShareState, isScreenAudioShared } from './react/features/screen-share/';
 import { toggleScreenshotCaptureEffect } from './react/features/screenshot-capture';
 import { setSharedVideoStatus } from './react/features/shared-video/actions';
 import { AudioMixerEffect } from './react/features/stream-effects/audio-mixer/AudioMixerEffect';
@@ -1546,6 +1548,8 @@ export default {
             this._desktopAudioStream = undefined;
         }
 
+        APP.store.dispatch(setScreenAudioShareState(false));
+
         if (didHaveVideo) {
             promise = promise.then(() => createLocalTracksF({ devices: [ 'video' ] }))
                 .then(([ stream ]) => {
@@ -1662,6 +1666,23 @@ export default {
                 = this._turnScreenSharingOff.bind(this, didHaveVideo);
 
             const desktopVideoStream = desktopStreams.find(stream => stream.getType() === MEDIA_TYPE.VIDEO);
+            const dekstopAudioStream = desktopStreams.find(stream => stream.getType() === MEDIA_TYPE.AUDIO);
+
+            if (dekstopAudioStream) {
+                dekstopAudioStream.on(
+                    JitsiTrackEvents.LOCAL_TRACK_STOPPED,
+                    () => {
+                        logger.debug(`Local screensharing audio track stopped. ${this.isSharingScreen}`);
+
+                        // Handle case where screen share was stopped from  the browsers 'screen share in progress'
+                        // window. If audio screen sharing is stopped via the normal UX flow this point shouldn't
+                        // be reached.
+                        isScreenAudioShared(APP.store.getState())
+                            && this._untoggleScreenSharing
+                            && this._untoggleScreenSharing();
+                    }
+                );
+            }
 
             if (desktopVideoStream) {
                 desktopVideoStream.on(
@@ -1830,14 +1851,28 @@ export default {
 
         return this._createDesktopTrack(options)
             .then(async streams => {
-                const desktopVideoStream = streams.find(stream => stream.getType() === MEDIA_TYPE.VIDEO);
+                let desktopVideoStream = streams.find(stream => stream.getType() === MEDIA_TYPE.VIDEO);
+
+                this._desktopAudioStream = streams.find(stream => stream.getType() === MEDIA_TYPE.AUDIO);
+
+                const { audioOnly = false } = options;
+
+                // If we're in audio only mode dispose of the video track otherwise the screensharing state will be
+                // inconsistent.
+                if (audioOnly) {
+                    desktopVideoStream.dispose();
+                    desktopVideoStream = undefined;
+
+                    if (!this._desktopAudioStream) {
+                        return Promise.reject(AUDIO_ONLY_SCREEN_SHARE_NO_TRACK);
+                    }
+                }
 
                 if (desktopVideoStream) {
                     logger.debug(`_switchToScreenSharing is using ${desktopVideoStream} for useVideoStream`);
                     await this.useVideoStream(desktopVideoStream);
                 }
 
-                this._desktopAudioStream = streams.find(stream => stream.getType() === MEDIA_TYPE.AUDIO);
 
                 if (this._desktopAudioStream) {
                     // If there is a localAudio stream, mix in the desktop audio stream captured by the screen sharing
@@ -1850,7 +1885,9 @@ export default {
                         // If no local stream is present ( i.e. no input audio devices) we use the screen share audio
                         // stream as we would use a regular stream.
                         await this.useAudioStream(this._desktopAudioStream);
+
                     }
+                    APP.store.dispatch(setScreenAudioShareState(true));
                 }
             })
             .then(() => {
@@ -1918,6 +1955,9 @@ export default {
         } else if (error.name === JitsiTrackErrors.SCREENSHARING_GENERIC_ERROR) {
             descriptionKey = 'dialog.screenSharingFailed';
             titleKey = 'dialog.screenSharingFailedTitle';
+        } else if (error === AUDIO_ONLY_SCREEN_SHARE_NO_TRACK) {
+            descriptionKey = 'notify.screenShareNoAudio';
+            titleKey = 'notify.screenShareNoAudioTitle';
         }
 
         APP.UI.messageHandler.showError({
@@ -2409,7 +2449,9 @@ export default {
         });
 
         APP.UI.addListener(
-            UIEvents.TOGGLE_SCREENSHARING, this.toggleScreenSharing.bind(this)
+            UIEvents.TOGGLE_SCREENSHARING, audioOnly => {
+                this.toggleScreenSharing(undefined, { audioOnly });
+            }
         );
 
         /* eslint-disable max-params */

config.js

@@ -428,7 +428,7 @@ var config = {
     // toolbarButtons: [
     //    'microphone', 'camera', 'closedcaptions', 'desktop', 'embedmeeting', 'fullscreen',
     //    'fodeviceselection', 'hangup', 'profile', 'chat', 'recording',
-    //    'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand',
+    //    'livestreaming', 'etherpad', 'sharedvideo', 'shareaudio', 'settings', 'raisehand',
     //    'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts',
     //    'tileview', 'select-background', 'download', 'help', 'mute-everyone', 'mute-video-everyone', 'security'
     // ],

lang/main.json

@@ -511,6 +511,8 @@
         "passwordRemovedRemotely": "$t(lockRoomPasswordUppercase) removed by another participant",
         "passwordSetRemotely": "$t(lockRoomPasswordUppercase) set by another participant",
         "raisedHand": "{{name}} would like to speak.",
+        "screenShareNoAudio": " Share audio box was not checked in the window selection screen.",
+        "screenShareNoAudioTitle": "Share audio was not chcked",
         "somebody": "Somebody",
         "startSilentTitle": "You joined with no audio output!",
         "startSilentDescription": "Rejoin the meeting to enable audio",
@@ -752,6 +754,7 @@
             "remoteVideoMute": "Disable camera of participant",
             "security": "Security options",
             "Settings": "Toggle settings",
+            "shareaudio": "Share audio",
             "sharedvideo": "Toggle YouTube video sharing",
             "shareRoom": "Invite someone",
             "shareYourScreen": "Toggle screenshare",
@@ -811,6 +814,7 @@
         "raiseYourHand": "Raise your hand",
         "security": "Security options",
         "Settings": "Settings",
+        "shareaudio": "Share audio",
         "sharedvideo": "Share a YouTube video",
         "shareRoom": "Invite someone",
         "shortcuts": "View shortcuts",

modules/UI/UIErrors.js

@@ -8,3 +8,10 @@
  * @type {{FEEDBACK_REQUEST_IN_PROGRESS: string}}
  */
 export const FEEDBACK_REQUEST_IN_PROGRESS = 'FeedbackRequestInProgress';
+
+/**
+ * Indicated an attempted audio only screen share session with no audio track present
+ *
+ * @type {{AUDIO_ONLY_SCREEN_SHARE_NO_TRACK: string}}
+ */
+export const AUDIO_ONLY_SCREEN_SHARE_NO_TRACK = 'AudioOnlyScreenShareNoTrack';

react/features/app/reducers.any.js

@@ -44,6 +44,7 @@ import '../recent-list/reducer';
 import '../recording/reducer';
 import '../settings/reducer';
 import '../subtitles/reducer';
+import '../screen-share/reducer';
 import '../toolbox/reducer';
 import '../transcribing/reducer';
 import '../video-layout/reducer';

react/features/base/conference/middleware.web.js

@@ -32,7 +32,7 @@ MiddlewareRegistry.register(({ dispatch, getState }) => next => action => {
     }
     case TOGGLE_SCREENSHARING: {
         if (typeof APP === 'object') {
-            APP.UI.emitEvent(UIEvents.TOGGLE_SCREENSHARING);
+            APP.UI.emitEvent(UIEvents.TOGGLE_SCREENSHARING, action.audioOnly);
         }
 
         break;

react/features/base/config/constants.js

@@ -16,7 +16,7 @@ export const _CONFIG_STORE_PREFIX = 'config.js';
 export const TOOLBAR_BUTTONS = [
     'microphone', 'camera', 'closedcaptions', 'desktop', 'embedmeeting', 'fullscreen',
     'fodeviceselection', 'hangup', 'profile', 'chat', 'recording',
-    'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand',
+    'livestreaming', 'etherpad', 'sharedvideo', 'shareaudio', 'settings', 'raisehand',
     'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts',
     'tileview', 'select-background', 'download', 'help', 'mute-everyone', 'mute-video-everyone',
     'security', 'toggle-camera'

react/features/base/environment/environment.js

@@ -41,6 +41,15 @@ export function isBrowsersOptimal(browserName: string) {
         .includes(browserName);
 }
 
+/**
+ * Returns whether or not the current OS is Mac.
+ *
+ * @returns {boolean}
+ */
+export function isMacOS() {
+    return Platform.OS === 'macos';
+}
+
 /**
  * Returns whether or not the current browser or the list of passed in browsers
  * is considered suboptimal. Suboptimal means it is a supported browser but has

react/features/base/icons/svg/index.js

@@ -99,6 +99,7 @@ export { default as IconSignalLevel0 } from './signal_cellular_0.svg';
 export { default as IconSignalLevel1 } from './signal_cellular_1.svg';
 export { default as IconSignalLevel2 } from './signal_cellular_2.svg';
 export { default as IconShare } from './share.svg';
+export { default as IconShareAudio } from './share-audio.svg';
 export { default as IconShareDesktop } from './share-desktop.svg';
 export { default as IconShareDoc } from './share-doc.svg';
 export { default as IconShareVideo } from './shared-video.svg';

react/features/base/icons/svg/share-audio.svg

@@ -0,0 +1,3 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M9.31658 3.06949L4.99999 6.66664H2.49999C2.03975 6.66664 1.66666 7.03974 1.66666 7.49998V12.5C1.66666 12.9602 2.03975 13.3333 2.49999 13.3333H4.99999L9.31658 16.9305C9.39146 16.9929 9.48585 17.027 9.58332 17.027C9.81344 17.027 9.99999 16.8405 9.99999 16.6104V3.38958C9.99999 3.2921 9.96582 3.19772 9.90341 3.12283C9.7561 2.94605 9.49336 2.92217 9.31658 3.06949ZM3.33332 8.33331H5.60341L8.33332 6.05838V13.9416L5.60341 11.6666H3.33332V8.33331ZM11.6667 6.66664C13.5076 6.66664 15 8.15903 15 9.99998C15 11.8409 13.5076 13.3333 11.6667 13.3333V11.6666C12.5871 11.6666 13.3333 10.9205 13.3333 9.99998C13.3333 9.0795 12.5871 8.33331 11.6667 8.33331V6.66664ZM11.6667 3.33331C15.3486 3.33331 18.3333 6.31808 18.3333 9.99998C18.3333 13.6819 15.3486 16.6666 11.6667 16.6666V15C14.4281 15 16.6667 12.7614 16.6667 9.99998C16.6667 7.23855 14.4281 4.99998 11.6667 4.99998V3.33331Z" fill="white"/>
+</svg>

react/features/base/tracks/actions.js

@@ -257,13 +257,16 @@ export function showNoDataFromSourceVideoError(jitsiTrack) {
  * Signals that the local participant is ending screensharing or beginning the
  * screensharing flow.
  *
+ * @param {boolean} audioOnly - Only share system audio.
  * @returns {{
  *     type: TOGGLE_SCREENSHARING,
+ *     audioOnly: boolean
  * }}
  */
-export function toggleScreensharing() {
+export function toggleScreensharing(audioOnly = false) {
     return {
-        type: TOGGLE_SCREENSHARING
+        type: TOGGLE_SCREENSHARING,
+        audioOnly
     };
 }
 

react/features/base/tracks/middleware.js

@@ -135,7 +135,7 @@ MiddlewareRegistry.register(store => next => action => {
 
     case TOGGLE_SCREENSHARING:
         if (typeof APP === 'object') {
-            APP.UI.emitEvent(UIEvents.TOGGLE_SCREENSHARING);
+            APP.UI.emitEvent(UIEvents.TOGGLE_SCREENSHARING, action.audioOnly);
         }
         break;
 

react/features/screen-share/actionTypes.js

@@ -0,0 +1,12 @@
+// @flow
+
+/**
+ * Type of action which sets the current state of screen audio sharing.
+ *
+ * {
+ *     type: SET_SCREEN_AUDIO_SHARE_STATE,
+ *     isSharingAudio: boolean
+ * }
+ */
+export const SET_SCREEN_AUDIO_SHARE_STATE = 'SET_SCREEN_AUDIO_SHARE_STATE';
+

react/features/screen-share/actions.js

@@ -0,0 +1,19 @@
+// @flow
+
+import { SET_SCREEN_AUDIO_SHARE_STATE } from './actionTypes';
+
+/**
+ * Updates the current known status of the shared video.
+ *
+ * @param {boolean} isSharingAudio - Is audio currently being shared or not.
+ * @returns {{
+ *     type: SET_SCREEN_AUDIO_SHARE_STATE,
+ *     isSharingAudio: boolean
+ * }}
+ */
+export function setScreenAudioShareState(isSharingAudio: boolean) {
+    return {
+        type: SET_SCREEN_AUDIO_SHARE_STATE,
+        isSharingAudio
+    };
+}

react/features/screen-share/functions.js

@@ -0,0 +1,25 @@
+// @flow
+
+import { isMacOS } from '../base/environment';
+import { browser } from '../base/lib-jitsi-meet';
+
+
+/**
+ * State of audio sharing.
+ *
+ * @param {Object} state - The state of the application.
+ * @returns {boolean}
+ */
+export function isScreenAudioShared(state: Object) {
+    return state['features/screen-share'].isSharingAudio;
+}
+
+/**
+ * Returns the visibility of the audio only screen share button. Currently electron on mac os doesn't
+ * have support for this functionality.
+ *
+ * @returns {boolean}
+ */
+export function isScreenAudioSupported() {
+    return !(browser.isElectron() && isMacOS());
+}

react/features/screen-share/index.js

@@ -0,0 +1,4 @@
+export * from './actions';
+export * from './actionTypes';
+export * from './functions';
+

react/features/screen-share/reducer.js

@@ -0,0 +1,22 @@
+
+import { ReducerRegistry } from '../base/redux';
+
+import { SET_SCREEN_AUDIO_SHARE_STATE } from './actionTypes';
+
+/**
+ * Reduces the Redux actions of the feature features/screen-share.
+ */
+ReducerRegistry.register('features/screen-share', (state = {}, action) => {
+    const { isSharingAudio } = action;
+
+    switch (action.type) {
+    case SET_SCREEN_AUDIO_SHARE_STATE:
+        return {
+            ...state,
+            isSharingAudio
+        };
+
+    default:
+        return state;
+    }
+});

react/features/stream-effects/audio-mixer/AudioMixerEffect.js

@@ -14,6 +14,16 @@ export class AudioMixerEffect {
      */
     _mixAudio: Object;
 
+    /**
+     * MediaStream resulted from mixing.
+     */
+    _mixedMediaStream: Object;
+
+    /**
+     * MediaStreamTrack obtained from mixed stream.
+     */
+    _mixedMediaTrack: Object;
+
     /**
      * Original MediaStream from the JitsiLocalTrack that uses this effect.
      */
@@ -68,7 +78,14 @@ export class AudioMixerEffect {
         this._audioMixer.addMediaStream(this._mixAudio.getOriginalStream());
         this._audioMixer.addMediaStream(this._originalStream);
 
-        return this._audioMixer.start();
+        this._mixedMediaStream = this._audioMixer.start();
+        this._mixedMediaTrack = this._mixedMediaStream.getTracks()[0];
+
+        // Sync the resulting mixed track enabled state with that of the track using the effect.
+        this.setMuted(!this._originalTrack.enabled);
+        this._originalTrack.enabled = true;
+
+        return this._mixedMediaStream;
     }
 
     /**
@@ -77,6 +94,9 @@ export class AudioMixerEffect {
      * @returns {void}
      */
     stopEffect() {
+        // Match state of the original track with that of the mixer track, not doing so can
+        // result in an inconsistent state e.g. redux state is muted yet track is enabled.
+        this._originalTrack.enabled = this._mixedMediaTrack.enabled;
         this._audioMixer.reset();
     }
 
@@ -87,7 +107,7 @@ export class AudioMixerEffect {
      * @returns {void}
      */
     setMuted(muted: boolean) {
-        this._originalTrack.enabled = !muted;
+        this._mixedMediaTrack.enabled = !muted;
     }
 
     /**
@@ -96,6 +116,6 @@ export class AudioMixerEffect {
      * @returns {boolean}
      */
     isMuted() {
-        return !this._originalTrack.enabled;
+        return !this._mixedMediaTrack.enabled;
     }
 }

react/features/toolbox/components/web/Toolbox.js

@@ -23,6 +23,7 @@ import {
     IconPresentation,
     IconRaisedHand,
     IconRec,
+    IconShareAudio,
     IconShareDesktop
 } from '../../../base/icons';
 import JitsiMeetJS from '../../../base/lib-jitsi-meet';
@@ -47,6 +48,7 @@ import {
     LiveStreamButton,
     RecordButton
 } from '../../../recording';
+import { isScreenAudioShared, isScreenAudioSupported } from '../../../screen-share/';
 import SecurityDialogButton from '../../../security/components/security-dialog/SecurityDialogButton';
 import {
     SETTINGS_TABS,
@@ -252,6 +254,7 @@ class Toolbox extends Component<Props> {
         this._onToolbarToggleProfile = this._onToolbarToggleProfile.bind(this);
         this._onToolbarToggleRaiseHand = this._onToolbarToggleRaiseHand.bind(this);
         this._onToolbarToggleScreenshare = this._onToolbarToggleScreenshare.bind(this);
+        this._onToolbarToggleShareAudio = this._onToolbarToggleShareAudio.bind(this);
         this._onToolbarOpenLocalRecordingInfoDialog = this._onToolbarOpenLocalRecordingInfoDialog.bind(this);
         this._onShortcutToggleTileView = this._onShortcutToggleTileView.bind(this);
     }
@@ -487,11 +490,12 @@ class Toolbox extends Component<Props> {
      * Dispatches an action to toggle screensharing.
      *
      * @private
+     * @param {boolean} audioOnly - Only share system audio.
      * @returns {void}
      */
-    _doToggleScreenshare() {
+    _doToggleScreenshare(audioOnly = false) {
         if (this.props._desktopSharingEnabled) {
-            this.props.dispatch(toggleScreensharing());
+            this.props.dispatch(toggleScreensharing(audioOnly));
         }
     }
 
@@ -857,6 +861,18 @@ class Toolbox extends Component<Props> {
         this._doToggleScreenshare();
     }
 
+    _onToolbarToggleShareAudio: () => void;
+
+    /**
+     * Handles toggle share audio action.
+     *
+     * @returns {void}
+     */
+    _onToolbarToggleShareAudio() {
+        this._closeOverflowMenuIfOpen();
+        this._doToggleScreenshare(true);
+    }
+
     _onToolbarOpenLocalRecordingInfoDialog: () => void;
 
     /**
@@ -983,6 +999,13 @@ class Toolbox extends Component<Props> {
                 && <SharedVideoButton
                     key = 'sharedvideo'
                     showLabel = { true } />,
+            this._shouldShowButton('shareaudio') && isScreenAudioSupported()
+                && <OverflowMenuItem
+                    accessibilityLabel = { t('toolbar.accessibilityLabel.shareaudio') }
+                    icon = { IconShareAudio }
+                    key = 'shareaudio'
+                    onClick = { this._onToolbarToggleShareAudio }
+                    text = { t('toolbar.shareaudio') } />,
             this._shouldShowButton('etherpad')
                 && <SharedDocumentButton
                     key = 'etherpad'
@@ -1322,7 +1345,7 @@ function _mapStateToProps(state) {
         _locked: locked,
         _overflowMenuVisible: overflowMenuVisible,
         _raisedHand: localParticipant.raisedHand,
-        _screensharing: localVideo && localVideo.videoType === 'desktop',
+        _screensharing: (localVideo && localVideo.videoType === 'desktop') || isScreenAudioShared(state),
         _visible: isToolboxVisible(state),
         _visibleButtons: getToolbarButtons(state)
     };