ref(conference) Simplify track creation. (#13209)

* ref(conference) Simplify track creation.
If gUM fails, we do not have to retry gUM with mic only and camera only constraints. gUM has come a long way and this is not needed anymore.

* ref(conference) Filter tracks that are added to conference.

* squash: Address review comments

* fix(prejoin): Display the exact gUM error in prejoin.

* squash: Address review comments

conference.js

@@ -559,7 +559,7 @@ export default {
             );
         }
 
-        let tryCreateLocalTracks;
+        let tryCreateLocalTracks = Promise.resolve([]);
 
         // On Electron there is no permission prompt for granting permissions. That's why we don't need to
         // spend much time displaying the overlay screen. If GUM is not resolved within 15 seconds it will
@@ -600,76 +600,65 @@ export default {
 
                     return [];
                 });
-        } else if (!requestedAudio && !requestedVideo) {
-            // Resolve with no tracks
-            tryCreateLocalTracks = Promise.resolve([]);
-        } else {
+        } else if (requestedAudio || requestedVideo) {
             tryCreateLocalTracks = createLocalTracksF({
                 devices: initialDevices,
                 timeout,
                 firePermissionPromptIsShownEvent: true
             })
-                .catch(err => {
-                    if (requestedAudio && requestedVideo) {
-
-                        // Try audio only...
-                        errors.audioAndVideoError = err;
+            .catch(async error => {
+                if (error.name === JitsiTrackErrors.TIMEOUT && !browser.isElectron()) {
+                    errors.audioAndVideoError = error;
 
-                        if (err.name === JitsiTrackErrors.TIMEOUT && !browser.isElectron()) {
-                            // In this case we expect that the permission prompt is still visible. There is no point of
-                            // executing GUM with different source. Also at the time of writing the following
-                            // inconsistency have been noticed in some browsers - if the permissions prompt is visible
-                            // and another GUM is executed the prompt does not change its content but if the user
-                            // clicks allow the user action isassociated with the latest GUM call.
-                            errors.audioOnlyError = err;
-                            errors.videoOnlyError = err;
+                    return [];
+                }
 
-                            return [];
-                        }
+                // Retry with separate gUM calls.
+                const gUMPromises = [];
+                const tracks = [];
 
-                        return createLocalTracksF(audioOptions);
-                    } else if (requestedAudio && !requestedVideo) {
-                        errors.audioOnlyError = err;
+                if (requestedAudio) {
+                    gUMPromises.push(createLocalTracksF(audioOptions));
+                }
 
-                        return [];
-                    } else if (requestedVideo && !requestedAudio) {
-                        errors.videoOnlyError = err;
+                if (requestedVideo) {
+                    gUMPromises.push(createLocalTracksF({
+                        devices: [ MEDIA_TYPE.VIDEO ],
+                        timeout,
+                        firePermissionPromptIsShownEvent: true
+                    }));
+                }
 
-                        return [];
-                    }
-                    logger.error('Should never happen');
-                })
-                .catch(err => {
-                    // Log this just in case...
-                    if (!requestedAudio) {
-                        logger.error('The impossible just happened', err);
-                    }
-                    errors.audioOnlyError = err;
-
-                    // Try video only...
-                    return requestedVideo
-                        ? createLocalTracksF({
-                            devices: [ MEDIA_TYPE.VIDEO ],
-                            firePermissionPromptIsShownEvent: true
-                        })
-                        : [];
-                })
-                .catch(err => {
-                    // Log this just in case...
-                    if (!requestedVideo) {
-                        logger.error('The impossible just happened', err);
+                const results = await Promise.allSettled(gUMPromises);
+                let errorMsg;
+
+                results.forEach((result, idx) => {
+                    if (result.status === 'fulfilled') {
+                        tracks.push(result.value[0]);
+                    } else {
+                        errorMsg = result.reason;
+                        const isAudio = idx === 0;
+
+                        logger.error(`${isAudio ? 'Audio' : 'Video'} track creation failed with error ${errorMsg}`);
+                        if (isAudio) {
+                            errors.audioOnlyError = errorMsg;
+                        } else {
+                            errors.videoOnlyError = errorMsg;
+                        }
                     }
-                    errors.videoOnlyError = err;
-
-                    return [];
                 });
+
+                if (errors.audioOnlyError && errors.videoOnlyError) {
+                    errors.audioAndVideoError = errorMsg;
+                }
+
+                return tracks;
+            });
         }
 
-        // Hide the permissions prompt/overlay as soon as the tracks are
-        // created. Don't wait for the connection to be made, since in some
-        // cases, when auth is required, for instance, that won't happen until
-        // the user inputs their credentials, but the dialog would be
-        // overshadowed by the overlay.
+        // Hide the permissions prompt/overlay as soon as the tracks are created. Don't wait for the connection to
+        // be established, as in some cases like when auth is required, connection won't be established until the user
+        // inputs their credentials, but the dialog would be overshadowed by the overlay.
         tryCreateLocalTracks.then(tracks => {
             APP.store.dispatch(mediaPermissionPromptVisibilityChanged(false));
 
@@ -810,43 +799,51 @@ export default {
         const initialOptions = {
             startAudioOnly: config.startAudioOnly,
             startScreenSharing: config.startScreenSharing,
-            startWithAudioMuted: getStartWithAudioMuted(state)
-                || isUserInteractionRequiredForUnmute(state),
-            startWithVideoMuted: getStartWithVideoMuted(state)
-                || isUserInteractionRequiredForUnmute(state)
+            startWithAudioMuted: getStartWithAudioMuted(state) || isUserInteractionRequiredForUnmute(state),
+            startWithVideoMuted: getStartWithVideoMuted(state) || isUserInteractionRequiredForUnmute(state)
         };
 
         this.roomName = roomName;
 
         try {
-            // Initialize the device list first. This way, when creating tracks
-            // based on preferred devices, loose label matching can be done in
-            // cases where the exact ID match is no longer available, such as
-            // when the camera device has switched USB ports.
-            // when in startSilent mode we want to start with audio muted
+            // Initialize the device list first. This way, when creating tracks based on preferred devices, loose label
+            // matching can be done in cases where the exact ID match is no longer available, such as -
+            // 1. When the camera device has switched USB ports.
+            // 2. When in startSilent mode we want to start with audio muted
             await this._initDeviceList();
         } catch (error) {
             logger.warn('initial device list initialization failed', error);
         }
 
-        const handleStartAudioMuted = (options, tracks) => {
-            if (options.startWithAudioMuted) {
+        // Filter out the local tracks based on various config options, i.e., when user joins muted or is muted by
+        // focus. However, audio track will always be created even though it is not added to the conference since we
+        // want audio related features (noisy mic, talk while muted, etc.) to work even if the mic is muted.
+        const handleInitialTracks = (options, tracks) => {
+            let localTracks = tracks;
+
+            // No local tracks are added when user joins as a visitor.
+            if (iAmVisitor(state)) {
+                return [];
+            }
+            if (options.startWithAudioMuted || room?.isStartAudioMuted()) {
                 // Always add the track on Safari because of a known issue where audio playout doesn't happen
                 // if the user joins audio and video muted, i.e., if there is no local media capture.
                 if (browser.isWebKitBased()) {
                     this.muteAudio(true, true);
                 } else {
-                    return tracks.filter(track => track.getType() !== MEDIA_TYPE.AUDIO);
+                    localTracks = localTracks.filter(track => track.getType() !== MEDIA_TYPE.AUDIO);
                 }
             }
+            if (room?.isStartVideoMuted()) {
+                localTracks = localTracks.filter(track => track.getType() !== MEDIA_TYPE.VIDEO);
+            }
 
-            return tracks;
+            return localTracks;
         };
 
         if (isPrejoinPageVisible(state)) {
             _connectionPromise = connect(roomName).then(c => {
-                // we want to initialize it early, in case of errors to be able
-                // to gather logs
+                // We want to initialize it early, in case of errors to be able to gather logs.
                 APP.connection = c;
 
                 return c;
@@ -859,48 +856,28 @@ export default {
             APP.store.dispatch(makePrecallTest(this._getConferenceOptions()));
 
             const { tryCreateLocalTracks, errors } = this.createInitialLocalTracks(initialOptions);
-            const tracks = await tryCreateLocalTracks;
+            const localTracks = await tryCreateLocalTracks;
 
-            // Initialize device list a second time to ensure device labels
-            // get populated in case of an initial gUM acceptance; otherwise
-            // they may remain as empty strings.
+            // Initialize device list a second time to ensure device labels get populated in case of an initial gUM
+            // acceptance; otherwise they may remain as empty strings.
             this._initDeviceList(true);
 
             if (isPrejoinPageVisible(state)) {
-                return APP.store.dispatch(initPrejoin(tracks, errors));
+                return APP.store.dispatch(initPrejoin(localTracks, errors));
             }
 
             logger.debug('Prejoin screen no longer displayed at the time when tracks were created');
 
             this._displayErrorsForCreateInitialLocalTracks(errors);
 
-            let localTracks = handleStartAudioMuted(initialOptions, tracks);
-
-            // In case where gUM is slow and resolves after the startAudio/VideoMuted coming from jicofo, we can be
-            // join unmuted even though jicofo had instruct us to mute, so let's respect that before passing the tracks
-            if (!browser.isWebKitBased()) {
-                if (room?.isStartAudioMuted()) {
-                    localTracks = localTracks.filter(track => track.getType() !== MEDIA_TYPE.AUDIO);
-                }
-            }
-
-            if (room?.isStartVideoMuted()) {
-                localTracks = localTracks.filter(track => track.getType() !== MEDIA_TYPE.VIDEO);
-            }
-
-            // Do not add the tracks if the user has joined the call as a visitor.
-            if (iAmVisitor(state)) {
-                return Promise.resolve();
-            }
-
-            return this._setLocalAudioVideoStreams(localTracks);
+            return this._setLocalAudioVideoStreams(handleInitialTracks(initialOptions, localTracks));
         }
 
         const [ tracks, con ] = await this.createInitialLocalTracksAndConnect(roomName, initialOptions);
 
         this._initDeviceList(true);
 
-        return this.startConference(con, handleStartAudioMuted(initialOptions, tracks));
+        return this.startConference(con, handleInitialTracks(initialOptions, tracks));
     },
 
     /**

react/features/base/tracks/functions.web.ts

@@ -1,8 +1,10 @@
 import { IStore } from '../../app/types';
 import { IStateful } from '../app/types';
 import { isMobileBrowser } from '../environment/utils';
-import JitsiMeetJS from '../lib-jitsi-meet';
+import JitsiMeetJS, { JitsiTrackErrors, browser } from '../lib-jitsi-meet';
 import { setAudioMuted } from '../media/actions';
+import { MEDIA_TYPE } from '../media/constants';
+import { getStartWithAudioMuted } from '../media/functions';
 import { toState } from '../redux/functions';
 import {
     getUserSelectedCameraDeviceId,
@@ -94,10 +96,10 @@ export function createLocalTracksF(options: ITrackOptions = {}, store?: IStore)
 }
 
 /**
- * Returns an object containing a promise which resolves with the created tracks &
- * the errors resulting from that process.
+ * Returns an object containing a promise which resolves with the created tracks and the errors resulting from that
+ * process.
  *
- * @returns {Promise<JitsiLocalTrack>}
+ * @returns {Promise<JitsiLocalTrack[]>}
  *
  * @todo Refactor to not use APP.
  */
@@ -106,7 +108,13 @@ export function createPrejoinTracks() {
     const initialDevices = [ 'audio' ];
     const requestedAudio = true;
     let requestedVideo = false;
-    const { startAudioOnly, startWithAudioMuted, startWithVideoMuted } = APP.store.getState()['features/base/settings'];
+    const { startAudioOnly, startWithVideoMuted } = APP.store.getState()['features/base/settings'];
+    const startWithAudioMuted = getStartWithAudioMuted(APP.store.getState());
+
+    // On Electron there is no permission prompt for granting permissions. That's why we don't need to
+    // spend much time displaying the overlay screen. If GUM is not resolved within 15 seconds it will
+    // probably never resolve.
+    const timeout = browser.isElectron() ? 15000 : 60000;
 
     // Always get a handle on the audio input device so that we have statistics even if the user joins the
     // conference muted. Previous implementation would only acquire the handle when the user first unmuted,
@@ -121,62 +129,66 @@ export function createPrejoinTracks() {
         requestedVideo = true;
     }
 
-    let tryCreateLocalTracks;
+    let tryCreateLocalTracks: any = Promise.resolve([]);
 
-    if (!requestedAudio && !requestedVideo) {
-        // Resolve with no tracks
-        tryCreateLocalTracks = Promise.resolve([]);
-    } else {
+    if (requestedAudio || requestedVideo) {
         tryCreateLocalTracks = createLocalTracksF({
             devices: initialDevices,
-            firePermissionPromptIsShownEvent: true
+            firePermissionPromptIsShownEvent: true,
+            timeout
         }, APP.store)
-                .catch((err: Error) => {
-                    if (requestedAudio && requestedVideo) {
+        .catch(async (err: Error) => {
+            if (err.name === JitsiTrackErrors.TIMEOUT && !browser.isElectron()) {
+                errors.audioAndVideoError = err;
 
-                        // Try audio only...
-                        errors.audioAndVideoError = err;
+                return [];
+            }
 
-                        return (
-                            createLocalTracksF({
-                                devices: [ 'audio' ],
-                                firePermissionPromptIsShownEvent: true
-                            }));
-                    } else if (requestedAudio && !requestedVideo) {
-                        errors.audioOnlyError = err;
+            // Retry with separate gUM calls.
+            const gUMPromises: any = [];
+            const tracks: any = [];
 
-                        return [];
-                    } else if (requestedVideo && !requestedAudio) {
-                        errors.videoOnlyError = err;
+            if (requestedAudio) {
+                gUMPromises.push(createLocalTracksF({
+                    devices: [ MEDIA_TYPE.AUDIO ],
+                    firePermissionPromptIsShownEvent: true,
+                    timeout
+                }));
+            }
 
-                        return [];
-                    }
-                    logger.error('Should never happen');
-                })
-                .catch((err: Error) => {
-                    // Log this just in case...
-                    if (!requestedAudio) {
-                        logger.error('The impossible just happened', err);
-                    }
-                    errors.audioOnlyError = err;
-
-                    // Try video only...
-                    return requestedVideo
-                        ? createLocalTracksF({
-                            devices: [ 'video' ],
-                            firePermissionPromptIsShownEvent: true
-                        })
-                        : [];
-                })
-                .catch((err: Error) => {
-                    // Log this just in case...
-                    if (!requestedVideo) {
-                        logger.error('The impossible just happened', err);
+            if (requestedVideo) {
+                gUMPromises.push(createLocalTracksF({
+                    devices: [ MEDIA_TYPE.VIDEO ],
+                    firePermissionPromptIsShownEvent: true,
+                    timeout
+                }));
+            }
+
+            const results = await Promise.allSettled(gUMPromises);
+            let errorMsg;
+
+            results.forEach((result, idx) => {
+                if (result.status === 'fulfilled') {
+                    tracks.push(result.value[0]);
+                } else {
+                    errorMsg = result.reason;
+                    const isAudio = idx === 0;
+
+                    logger.error(`${isAudio ? 'Audio' : 'Video'} track creation failed with error ${errorMsg}`);
+                    if (isAudio) {
+                        errors.audioOnlyError = errorMsg;
+                    } else {
+                        errors.videoOnlyError = errorMsg;
                     }
-                    errors.videoOnlyError = err;
+                }
+            });
+
+            if (errors.audioOnlyError && errors.videoOnlyError) {
+                errors.audioAndVideoError = errorMsg;
+            }
 
-                    return [];
-                });
+            return tracks;
+        });
     }
 
     return {

react/features/prejoin/reducer.ts

@@ -165,31 +165,31 @@ ReducerRegistry.register<IPrejoinState>(
 function getStatusFromErrors(errors: {
     audioAndVideoError?: { message: string; };
     audioOnlyError?: { message: string; };
-    videoOnlyError?: Object; }
+    videoOnlyError?: { message: string; }; }
 ) {
     const { audioOnlyError, videoOnlyError, audioAndVideoError } = errors;
 
     if (audioAndVideoError) {
-        if (audioOnlyError) {
-            if (videoOnlyError) {
-                return {
-                    deviceStatusType: 'warning',
-                    deviceStatusText: 'prejoin.audioAndVideoError',
-                    rawError: audioAndVideoError.message
-                };
-            }
+        return {
+            deviceStatusType: 'warning',
+            deviceStatusText: 'prejoin.audioAndVideoError',
+            rawError: audioAndVideoError.message
+        };
+    }
 
-            return {
-                deviceStatusType: 'warning',
-                deviceStatusText: 'prejoin.audioOnlyError',
-                rawError: audioOnlyError.message
-            };
-        }
+    if (audioOnlyError) {
+        return {
+            deviceStatusType: 'warning',
+            deviceStatusText: 'prejoin.audioOnlyError',
+            rawError: audioOnlyError.message
+        };
+    }
 
+    if (videoOnlyError) {
         return {
             deviceStatusType: 'warning',
             deviceStatusText: 'prejoin.videoOnlyError',
-            rawError: audioAndVideoError.message
+            rawError: videoOnlyError.message
         };
     }