feat(virtual-background): Desktop share as virtual background wip

css/modals/virtual-background/_virtual-background.scss

@@ -7,7 +7,7 @@
     grid-template-columns: auto auto auto auto auto;
     column-gap: 9px;
     cursor: pointer;
-    .thumbnail:hover, .blur:hover, .slight-blur:hover, .virtual-background-none:hover{
+    .desktop-share:hover, .thumbnail:hover, .blur:hover, .slight-blur:hover, .virtual-background-none:hover{
         height: 56px;
         width: 103px;
         opacity: .5;
@@ -113,14 +113,43 @@
         vertical-align: middle;
         line-height: 60px;
     }
+    .desktop-share{
+        margin-top: 8px;
+        background: #525252;
+        font-weight: bold;
+        height: 60px;
+        width: 107px;
+        border-radius: 6px;
+        text-align: center;
+        vertical-align: middle;
+        line-height: 60px;
+    }
+    .desktop-share-selected{
+        margin-top: 8px;
+        background: #525252;
+        font-weight: bold;
+        height: 56px;
+        width: 103px;
+        border-radius: 6px;
+        border: 2px solid #246FE5;
+        text-align: center;
+        vertical-align: middle;
+        line-height: 60px;
+    }
+    .share-desktop-icon{
+        margin-top: 15%;
+    }
 
     @media (min-width: 432px) and (max-width: 632px) {
         font-size: 1.5vw;
-        .virtual-background-none, .thumbnail, .blur, .slight-blur{
+        .share-desktop-icon{
+            margin-top: 25%;
+        }
+        .desktop-share, .virtual-background-none, .thumbnail, .blur, .slight-blur{
             height: 60px;
             width: 60px;
         }
-        .thumbnail-selected, .none-selected, .blur-selected, .slight-blur-selected{
+        .desktop-share-selected, .thumbnail-selected, .none-selected, .blur-selected, .slight-blur-selected{
             height: 56px;
             width: 56px;
         }

react/features/stream-effects/virtual-background/JitsiStreamBackgroundEffect.js

@@ -1,4 +1,6 @@
 // @flow
+import { JitsiTrackEvents } from '../../base/lib-jitsi-meet';
+
 import {
     CLEAR_TIMEOUT,
     TIMEOUT_TICK,
@@ -14,6 +16,7 @@ import {
 export default class JitsiStreamBackgroundEffect {
     _model: Object;
     _options: Object;
+    _stream: Object;
     _segmentationPixelCount: number;
     _inputVideoElement: HTMLVideoElement;
     _onMaskFrameTimer: Function;
@@ -25,6 +28,7 @@ export default class JitsiStreamBackgroundEffect {
     _segmentationMaskCanvas: Object;
     _renderMask: Function;
     _virtualImage: HTMLImageElement;
+    _virtualVideo: HTMLVideoElement;
     isEnabled: Function;
     startEffect: Function;
     stopEffect: Function;
@@ -35,15 +39,22 @@ export default class JitsiStreamBackgroundEffect {
      * @class
      * @param {Object} model - Meet model.
      * @param {Object} options - Segmentation dimensions.
+     * @param {Object} screenSharing - Desktop track for displaying desktop share as virtual background.
      */
-    constructor(model: Object, options: Object) {
+    constructor(model: Object, options: Object, screenSharing: Object) {
         this._options = options;
+        this._stream = screenSharing;
 
         if (this._options.virtualBackground.backgroundType === 'image') {
             this._virtualImage = document.createElement('img');
             this._virtualImage.crossOrigin = 'anonymous';
             this._virtualImage.src = this._options.virtualBackground.virtualSource;
         }
+        if (this._options.virtualBackground.backgroundType === 'desktop-share' && this._stream) {
+            this._virtualVideo = document.createElement('video');
+            this._virtualVideo.autoplay = true;
+            this._virtualVideo.srcObject = this._stream.stream;
+        }
         this._model = model;
         this._options = options;
         this._segmentationPixelCount = this._options.width * this._options.height;
@@ -119,6 +130,13 @@ export default class JitsiStreamBackgroundEffect {
                 this._inputVideoElement.width,
                 this._inputVideoElement.height
             );
+        }
+        if (this._options.virtualBackground.backgroundType === 'desktop-share') {
+            this._outputCanvasCtx.drawImage(
+                this._virtualVideo,
+                0,
+                0
+            );
         } else {
             this._outputCanvasCtx.filter = `blur(${this._options.virtualBackground.blurValue}px)`;
             this._outputCanvasCtx.drawImage(this._inputVideoElement, 0, 0);
@@ -234,6 +252,15 @@ export default class JitsiStreamBackgroundEffect {
         this._inputVideoElement.height = parseInt(height, 10);
         this._inputVideoElement.autoplay = true;
         this._inputVideoElement.srcObject = stream;
+        this._stream && this._stream.on(
+            JitsiTrackEvents.LOCAL_TRACK_STOPPED,
+            () => {
+                this._options.virtualBackground.enabled = false;
+                this._options.virtualBackground.backgroundType = 'none';
+                this._options.virtualBackground.selectedThumbnail = 'none';
+                this._options.virtualBackground.backgroundEffectEnabled = false;
+                this._options.virtualBackground.enabled = false;
+            });
         this._inputVideoElement.onloadeddata = () => {
             this._maskFrameTimerWorker.postMessage({
                 id: SET_TIMEOUT,

react/features/stream-effects/virtual-background/index.js

@@ -2,6 +2,8 @@
 
 import * as wasmCheck from 'wasm-check';
 
+import { createLocalTrack } from '../../base/lib-jitsi-meet/functions';
+
 import JitsiStreamBackgroundEffect from './JitsiStreamBackgroundEffect';
 import createTFLiteModule from './vendor/tflite/tflite';
 import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
@@ -35,6 +37,7 @@ export async function createVirtualBackgroundEffect(virtualBackground: Object) {
         throw new Error('JitsiStreamBackgroundEffect not supported!');
     }
     let tflite;
+    let screenSharing;
 
     if (wasmCheck.feature.simd) {
         tflite = await createTFLiteSIMDModule();
@@ -55,10 +58,14 @@ export async function createVirtualBackgroundEffect(virtualBackground: Object) {
 
     tflite._loadModel(model.byteLength);
 
+    if (virtualBackground.backgroundType === 'desktop-share') {
+        screenSharing = await createLocalTrack('desktop', '');
+    }
+
     const options = {
         ...wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96,
         virtualBackground
     };
 
-    return new JitsiStreamBackgroundEffect(tflite, options);
+    return new JitsiStreamBackgroundEffect(tflite, options, screenSharing);
 }

react/features/virtual-background/components/VirtualBackgroundDialog.js

@@ -7,7 +7,7 @@ import uuid from 'uuid';
 
 import { Dialog, hideDialog } from '../../base/dialog';
 import { translate } from '../../base/i18n';
-import { Icon, IconCloseSmall, IconPlusCircle } from '../../base/icons';
+import { Icon, IconCloseSmall, IconPlusCircle, IconShareDesktop } from '../../base/icons';
 import { connect } from '../../base/redux';
 import { getLocalVideoTrack } from '../../base/tracks';
 import { toggleBackgroundEffect } from '../actions';
@@ -119,6 +119,14 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, dispatch, t }: Pro
         });
     };
 
+    const shareDesktop = async selection => {
+        setOptions({
+            backgroundType: 'desktop-share',
+            enabled: true,
+            selectedThumbnail: selection
+        });
+    };
+
     const setUploadedImageBackground = async image => {
         setOptions({
             backgroundType: 'image',
@@ -222,6 +230,16 @@ function VirtualBackground({ _jitsiTrack, _selectedThumbnail, dispatch, t }: Pro
                             onClick = { () => enableBlur(25, 'blur') }>
                             {t('virtualBackground.blur')}
                         </div>
+                        <div
+                            className = { _selectedThumbnail === 'desktop-share'
+                                ? 'desktop-share-selected'
+                                : 'desktop-share' }
+                            onClick = { () => shareDesktop('desktop-share') }>
+                            <Icon
+                                className = 'share-desktop-icon'
+                                size = { 30 }
+                                src = { IconShareDesktop } />
+                        </div>
                         {images.map((image, index) => (
                             <img
                                 className = {

react/features/virtual-background/reducer.js

@@ -6,11 +6,6 @@ import { BACKGROUND_ENABLED, SET_VIRTUAL_BACKGROUND } from './actionTypes';
 
 const STORE_NAME = 'features/virtual-background';
 
-/**
- * Sets up the persistence of the feature {@code virtual-background}.
- */
-PersistenceRegistry.register(STORE_NAME, true);
-
 /**
  * Reduces redux actions which activate/deactivate virtual background image, or
  * indicate if the virtual image background is activated/deactivated. The
@@ -25,6 +20,11 @@ PersistenceRegistry.register(STORE_NAME, true);
 ReducerRegistry.register(STORE_NAME, (state = {}, action) => {
     const { virtualSource, backgroundEffectEnabled, blurValue, backgroundType, selectedThumbnail } = action;
 
+    /**
+     * Sets up the persistence of the feature {@code virtual-background}.
+     */
+    PersistenceRegistry.register(STORE_NAME, state.backgroundType !== 'desktop-share');
+
     switch (action.type) {
     case SET_VIRTUAL_BACKGROUND: {
         return {