Roshan Pulapura 3 years ago
parent
commit
09441c2632
No account linked to committer's email address

+ 3
- 6
react/features/stream-effects/virtual-background/JitsiStreamBackgroundEffect.js View File

166
         const outputMemoryOffset = this._model._getOutputMemoryOffset() / 4;
166
         const outputMemoryOffset = this._model._getOutputMemoryOffset() / 4;
167
 
167
 
168
         for (let i = 0; i < this._segmentationPixelCount; i++) {
168
         for (let i = 0; i < this._segmentationPixelCount; i++) {
169
-            const background = this._model.HEAPF32[outputMemoryOffset + (i * 2)];
170
-            const person = this._model.HEAPF32[outputMemoryOffset + (i * 2) + 1];
171
-            const shift = Math.max(background, person);
172
-            const backgroundExp = Math.exp(background - shift);
173
-            const personExp = Math.exp(person - shift);
169
+            const person = this._model.HEAPF32[outputMemoryOffset + i];
174
 
170
 
175
             // Sets only the alpha component of each pixel.
171
             // Sets only the alpha component of each pixel.
176
-            this._segmentationMask.data[(i * 4) + 3] = (255 * personExp) / (backgroundExp + personExp);
172
+            this._segmentationMask.data[(i * 4) + 3] = 255 * person;
173
+
177
         }
174
         }
178
         this._segmentationMaskCtx.putImageData(this._segmentationMask, 0, 0);
175
         this._segmentationMaskCtx.putImageData(this._segmentationMask, 0, 0);
179
     }
176
     }

+ 4
- 9
react/features/stream-effects/virtual-background/index.js View File

9
 import createTFLiteModule from './vendor/tflite/tflite';
9
 import createTFLiteModule from './vendor/tflite/tflite';
10
 import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
10
 import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
11
 const models = {
11
 const models = {
12
-    model96: 'libs/segm_lite_v681.tflite',
13
-    model144: 'libs/segm_full_v679.tflite'
12
+    modelLandscape: 'libs/selfie_segmentation_landscape.tflite'
14
 };
13
 };
15
 
14
 
16
 let tflite;
15
 let tflite;
18
 let isWasmDisabled = false;
17
 let isWasmDisabled = false;
19
 
18
 
20
 const segmentationDimensions = {
19
 const segmentationDimensions = {
21
-    model96: {
22
-        height: 96,
23
-        width: 160
24
-    },
25
-    model144: {
20
+    modelLandscape: {
26
         height: 144,
21
         height: 144,
27
         width: 256
22
         width: 256
28
     }
23
     }
83
     }
78
     }
84
 
79
 
85
     const modelBufferOffset = tflite._getModelBufferMemoryOffset();
80
     const modelBufferOffset = tflite._getModelBufferMemoryOffset();
86
-    const modelResponse = await fetch(wasmCheck.feature.simd ? models.model144 : models.model96);
81
+    const modelResponse = await fetch(models.modelLandscape);
87
 
82
 
88
     if (!modelResponse.ok) {
83
     if (!modelResponse.ok) {
89
         throw new Error('Failed to download tflite model!');
84
         throw new Error('Failed to download tflite model!');
96
     tflite._loadModel(model.byteLength);
91
     tflite._loadModel(model.byteLength);
97
 
92
 
98
     const options = {
93
     const options = {
99
-        ...wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96,
94
+        ...segmentationDimensions.modelLandscape,
100
         virtualBackground
95
         virtualBackground
101
     };
96
     };
102
 
97
 

+ 1
- 13
react/features/stream-effects/virtual-background/vendor/README.md View File

1
 # Virtual Background on stream effects
1
 # Virtual Background on stream effects
2
 
2
 
3
-> Inspired from https://ai.googleblog.com/2020/10/background-features-in-google-meet.html and https://github.com/Volcomix/virtual-background.git
3
+> From https://google.github.io/mediapipe/solutions/models.html#selfie-segmentation
4
 
4
 
5
 #### Canvas 2D + CPU
5
 #### Canvas 2D + CPU
6
 
6
 
22
 - [WebAssembly](https://webassembly.org/)
22
 - [WebAssembly](https://webassembly.org/)
23
 - [WebAssembly SIMD](https://github.com/WebAssembly/simd)
23
 - [WebAssembly SIMD](https://github.com/WebAssembly/simd)
24
 - [TFLite](https://blog.tensorflow.org/2020/07/accelerating-tensorflow-lite-xnnpack-integration.html)
24
 - [TFLite](https://blog.tensorflow.org/2020/07/accelerating-tensorflow-lite-xnnpack-integration.html)
25
-
26
-## LICENSE
27
-
28
-The mdoels vendored here were downloaded early January (they were available as early as the 4th), before Google switched the license away from Apache 2. Thus we understand they are not covered by the new license which according to the [model card](https://drive.google.com/file/d/1lnP1bRi9CSqQQXUHa13159vLELYDgDu0/view) dates from the 21st of January.
29
-
30
-We are not lawyers so do get legal advise if in doubt.
31
-
32
-References:
33
-
34
-- Model license discussion: https://github.com/tensorflow/tfjs/issues/4177
35
-- Current vendored model is discovered: https://github.com/tensorflow/tfjs/issues/4177#issuecomment-753934631
36
-- License change is noticed: https://github.com/tensorflow/tfjs/issues/4177#issuecomment-771536641

BIN
react/features/stream-effects/virtual-background/vendor/models/segm_full_v679.tflite View File


BIN
react/features/stream-effects/virtual-background/vendor/models/segm_lite_v681.tflite View File


BIN
react/features/stream-effects/virtual-background/vendor/models/selfie_segmentation_landscape.tflite View File


Loading…
Cancel
Save