feat(virtual-background): Desktop share as virtual background

react/features/stream-effects/virtual-background/JitsiStreamBackgroundEffect.js

@@ -1,5 +1,7 @@
 // @flow
 
+import { VIRTUAL_BACKGROUND_TYPE } from '../../virtual-background/constants';
+
 import {
     CLEAR_TIMEOUT,
     TIMEOUT_TICK,
@@ -15,6 +17,7 @@ import {
 export default class JitsiStreamBackgroundEffect {
     _model: Object;
     _options: Object;
+    _desktopShareDimensions: Object;
     _segmentationPixelCount: number;
     _inputVideoElement: HTMLVideoElement;
     _onMaskFrameTimer: Function;
@@ -41,18 +44,21 @@ export default class JitsiStreamBackgroundEffect {
     constructor(model: Object, options: Object) {
         this._options = options;
 
-        if (this._options.virtualBackground.backgroundType === 'image') {
+        if (this._options.virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
             this._virtualImage = document.createElement('img');
             this._virtualImage.crossOrigin = 'anonymous';
             this._virtualImage.src = this._options.virtualBackground.virtualSource;
         }
-        if (this._options.virtualBackground.backgroundType === 'desktop-share') {
+        if (this._options.virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.DESKTOP_SHARE) {
+            const desktopShareTrack = this._options?.virtualBackground?.virtualSource?.track;
+
             this._virtualVideo = document.createElement('video');
             this._virtualVideo.autoplay = true;
             this._virtualVideo.srcObject = this._options?.virtualBackground?.virtualSource?.stream;
+            this._desktopShareDimensions = desktopShareTrack.getSettings ? desktopShareTrack.getSettings()
+                : desktopShareTrack.getConstraints();
         }
         this._model = model;
-        this._options = options;
         this._segmentationPixelCount = this._options.width * this._options.height;
 
         // Bind event handler so it is only bound once for every instance.
@@ -89,7 +95,7 @@ export default class JitsiStreamBackgroundEffect {
         //
 
         // Smooth out the edges.
-        if (this._options.virtualBackground.backgroundType === 'image') {
+        if (this._options.virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
             this._outputCanvasCtx.filter = 'blur(4px)';
         } else {
             this._outputCanvasCtx.filter = 'blur(8px)';
@@ -118,7 +124,7 @@ export default class JitsiStreamBackgroundEffect {
         //
 
         this._outputCanvasCtx.globalCompositeOperation = 'destination-over';
-        if (this._options.virtualBackground.backgroundType === 'image') {
+        if (this._options.virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
             this._outputCanvasCtx.drawImage(
                 this._virtualImage,
                 0,
@@ -127,13 +133,13 @@ export default class JitsiStreamBackgroundEffect {
                 this._inputVideoElement.height
             );
         }
-        if (this._options.virtualBackground.backgroundType === 'desktop-share') {
+        if (this._options.virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.DESKTOP_SHARE) {
             this._outputCanvasCtx.drawImage(
                 this._virtualVideo,
                 0,
                 0,
-                this._inputVideoElement.width,
-                this._inputVideoElement.height
+                this._desktopShareDimensions.width,
+                this._desktopShareDimensions.height
             );
         } else {
             this._outputCanvasCtx.filter = `blur(${this._options.virtualBackground.blurValue}px)`;

react/features/toolbox/components/web/Toolbox.js

@@ -75,6 +75,7 @@ import {
 } from '../../../video-quality';
 import { VideoBackgroundButton } from '../../../virtual-background';
 import { toggleBackgroundEffect } from '../../../virtual-background/actions';
+import { VIRTUAL_BACKGROUND_TYPE } from '../../../virtual-background/constants';
 import { checkBlurSupport } from '../../../virtual-background/functions';
 import {
     setFullScreen,
@@ -907,11 +908,11 @@ class Toolbox extends Component<Props> {
      * @returns {void}
      */
     _onToolbarToggleScreenshare() {
-        if (this.props._backgroundType === 'desktop-share') {
+        if (this.props._backgroundType === VIRTUAL_BACKGROUND_TYPE.DESKTOP_SHARE) {
             const noneOptions = {
                 enabled: false,
-                backgroundType: 'none',
-                selectedThumbnail: 'none',
+                backgroundType: VIRTUAL_BACKGROUND_TYPE.NONE,
+                selectedThumbnail: VIRTUAL_BACKGROUND_TYPE.NONE,
                 backgroundEffectEnabled: false
             };
 

react/features/virtual-background/components/VirtualBackgroundDialog.js

@@ -9,9 +9,11 @@ import { Dialog, hideDialog } from '../../base/dialog';
 import { translate } from '../../base/i18n';
 import { Icon, IconCloseSmall, IconPlusCircle, IconShareDesktop } from '../../base/icons';
 import { createLocalTrack } from '../../base/lib-jitsi-meet/functions';
+import { VIDEO_TYPE } from '../../base/media';
 import { connect } from '../../base/redux';
 import { getLocalVideoTrack } from '../../base/tracks';
 import { toggleBackgroundEffect } from '../actions';
+import { VIRTUAL_BACKGROUND_TYPE } from '../constants';
 import { resizeImage, toDataURL } from '../functions';
 import logger from '../logger';
 
@@ -89,7 +91,7 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, _virtualSource, di
     const localImages = jitsiLocalStorage.getItem('virtualBackgrounds');
     const [ storedImages, setStoredImages ] = useState((localImages && JSON.parse(localImages)) || []);
     const [ loading, isloading ] = useState(false);
-    const [ activeDesktopVideo ] = useState(_virtualSource?.videoType === 'desktop' ? _virtualSource : null);
+    const [ activeDesktopVideo ] = useState(_virtualSource?.videoType === VIDEO_TYPE.DESKTOP ? _virtualSource : null);
 
     const deleteStoredImage = image => {
         setStoredImages(storedImages.filter(item => item !== image));
@@ -112,7 +114,7 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, _virtualSource, di
 
     const enableBlur = async (blurValue, selection) => {
         setOptions({
-            backgroundType: 'blur',
+            backgroundType: VIRTUAL_BACKGROUND_TYPE.BLUR,
             enabled: true,
             blurValue,
             selectedThumbnail: selection
@@ -129,8 +131,11 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, _virtualSource, di
     const shareDesktop = async selection => {
         const url = await createLocalTrack('desktop', '');
 
+        if (!url) {
+            throw new Error('Could not create desktop local track!');
+        }
         setOptions({
-            backgroundType: 'desktop-share',
+            backgroundType: VIRTUAL_BACKGROUND_TYPE.DESKTOP_SHARE,
             enabled: true,
             selectedThumbnail: selection,
             url
@@ -139,7 +144,7 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, _virtualSource, di
 
     const setUploadedImageBackground = async image => {
         setOptions({
-            backgroundType: 'image',
+            backgroundType: VIRTUAL_BACKGROUND_TYPE.IMAGE,
             enabled: true,
             url: image.src,
             selectedThumbnail: image.id
@@ -150,7 +155,7 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, _virtualSource, di
         const url = await toDataURL(image.src);
 
         setOptions({
-            backgroundType: 'image',
+            backgroundType: VIRTUAL_BACKGROUND_TYPE.IMAGE,
             enabled: true,
             url,
             selectedThumbnail: image.id
@@ -173,7 +178,7 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, _virtualSource, di
                 }
             ]);
             setOptions({
-                backgroundType: 'image',
+                backgroundType: VIRTUAL_BACKGROUND_TYPE.IMAGE,
                 enabled: true,
                 url,
                 selectedThumbnail: uuId

react/features/virtual-background/components/VirtualBackgroundPreview.js

@@ -4,11 +4,13 @@ import Spinner from '@atlaskit/spinner';
 import React, { PureComponent } from 'react';
 
 import { translate } from '../../base/i18n';
+import { VIDEO_TYPE } from '../../base/media';
 import Video from '../../base/media/components/Video';
 import { connect, equals } from '../../base/redux';
 import { getCurrentCameraDeviceId } from '../../base/settings';
 import { createLocalTracksF } from '../../base/tracks/functions';
 import { toggleBackgroundEffect } from '../actions';
+import { VIRTUAL_BACKGROUND_TYPE } from '../constants';
 import { localTrackStopped } from '../functions';
 
 const videoClassName = 'video-preview-video';
@@ -207,12 +209,12 @@ class VirtualBackgroundPreview extends PureComponent<Props, State> {
             this._setTracks();
         }
         if (!equals(this.props.options, prevProps.options)) {
-            if (prevProps.options.backgroundType === 'desktop-share') {
+            if (prevProps.options.backgroundType === VIRTUAL_BACKGROUND_TYPE.DESKTOP_SHARE) {
                 prevProps.options.url.dispose();
             }
             this._applyBackgroundEffect();
         }
-        if (this.props.options.url?.videoType === 'desktop') {
+        if (this.props.options.url?.videoType === VIDEO_TYPE.DESKTOP) {
             localTrackStopped(this.props.dispatch, this.props.options.url, this.state.jitsiTrack);
         }
     }

react/features/virtual-background/constants.js

@@ -0,0 +1,11 @@
+/**
+ * An enumeration of the different virtual background types.
+ *
+ * @enum {string}
+ */
+export const VIRTUAL_BACKGROUND_TYPE = {
+    IMAGE: 'image',
+    DESKTOP_SHARE: 'desktop-share',
+    BLUR: 'blur',
+    NONE: 'none'
+};

react/features/virtual-background/middleware.js

@@ -1,5 +1,6 @@
 // @flow
 
+import { VIDEO_TYPE } from '../base/media';
 import { MiddlewareRegistry } from '../base/redux';
 import { getLocalVideoTrack } from '../base/tracks';
 
@@ -18,7 +19,7 @@ MiddlewareRegistry.register(store => next => action => {
     const virtualSource = getState()['features/virtual-background'].virtualSource;
     const currentLocalTrack = getLocalVideoTrack(getState()['features/base/tracks']);
 
-    if (virtualSource?.videoType === 'desktop') {
+    if (virtualSource?.videoType === VIDEO_TYPE.DESKTOP && currentLocalTrack) {
         localTrackStopped(dispatch, virtualSource, currentLocalTrack.jitsiTrack);
     }
 

react/features/virtual-background/reducer.js

@@ -3,6 +3,7 @@
 import { PersistenceRegistry, ReducerRegistry } from '../base/redux';
 
 import { BACKGROUND_ENABLED, SET_VIRTUAL_BACKGROUND } from './actionTypes';
+import { VIRTUAL_BACKGROUND_TYPE } from './constants';
 
 const STORE_NAME = 'features/virtual-background';
 
@@ -23,7 +24,7 @@ ReducerRegistry.register(STORE_NAME, (state = {}, action) => {
     /**
      * Sets up the persistence of the feature {@code virtual-background}.
      */
-    PersistenceRegistry.register(STORE_NAME, state.backgroundType !== 'desktop-share');
+    PersistenceRegistry.register(STORE_NAME, state.backgroundType !== VIRTUAL_BACKGROUND_TYPE.DESKTOP_SHARE);
 
     switch (action.type) {
     case SET_VIRTUAL_BACKGROUND: {