浏览代码

feat(virtual-backgrounds) use new Open Source model

https://google.github.io/mediapipe/solutions/models.html#selfie-segmentation
factor2
Roshan Pulapura 3 年前
父节点
当前提交
09441c2632
没有帐户链接到提交者的电子邮件

+ 3
- 6
react/features/stream-effects/virtual-background/JitsiStreamBackgroundEffect.js 查看文件

166
         const outputMemoryOffset = this._model._getOutputMemoryOffset() / 4;
166
         const outputMemoryOffset = this._model._getOutputMemoryOffset() / 4;
167
 
167
 
168
         for (let i = 0; i < this._segmentationPixelCount; i++) {
168
         for (let i = 0; i < this._segmentationPixelCount; i++) {
169
-            const background = this._model.HEAPF32[outputMemoryOffset + (i * 2)];
170
-            const person = this._model.HEAPF32[outputMemoryOffset + (i * 2) + 1];
171
-            const shift = Math.max(background, person);
172
-            const backgroundExp = Math.exp(background - shift);
173
-            const personExp = Math.exp(person - shift);
169
+            const person = this._model.HEAPF32[outputMemoryOffset + i];
174
 
170
 
175
             // Sets only the alpha component of each pixel.
171
             // Sets only the alpha component of each pixel.
176
-            this._segmentationMask.data[(i * 4) + 3] = (255 * personExp) / (backgroundExp + personExp);
172
+            this._segmentationMask.data[(i * 4) + 3] = 255 * person;
173
+
177
         }
174
         }
178
         this._segmentationMaskCtx.putImageData(this._segmentationMask, 0, 0);
175
         this._segmentationMaskCtx.putImageData(this._segmentationMask, 0, 0);
179
     }
176
     }

+ 4
- 9
react/features/stream-effects/virtual-background/index.js 查看文件

9
 import createTFLiteModule from './vendor/tflite/tflite';
9
 import createTFLiteModule from './vendor/tflite/tflite';
10
 import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
10
 import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
11
 const models = {
11
 const models = {
12
-    model96: 'libs/segm_lite_v681.tflite',
13
-    model144: 'libs/segm_full_v679.tflite'
12
+    modelLandscape: 'libs/selfie_segmentation_landscape.tflite'
14
 };
13
 };
15
 
14
 
16
 let tflite;
15
 let tflite;
18
 let isWasmDisabled = false;
17
 let isWasmDisabled = false;
19
 
18
 
20
 const segmentationDimensions = {
19
 const segmentationDimensions = {
21
-    model96: {
22
-        height: 96,
23
-        width: 160
24
-    },
25
-    model144: {
20
+    modelLandscape: {
26
         height: 144,
21
         height: 144,
27
         width: 256
22
         width: 256
28
     }
23
     }
83
     }
78
     }
84
 
79
 
85
     const modelBufferOffset = tflite._getModelBufferMemoryOffset();
80
     const modelBufferOffset = tflite._getModelBufferMemoryOffset();
86
-    const modelResponse = await fetch(wasmCheck.feature.simd ? models.model144 : models.model96);
81
+    const modelResponse = await fetch(models.modelLandscape);
87
 
82
 
88
     if (!modelResponse.ok) {
83
     if (!modelResponse.ok) {
89
         throw new Error('Failed to download tflite model!');
84
         throw new Error('Failed to download tflite model!');
96
     tflite._loadModel(model.byteLength);
91
     tflite._loadModel(model.byteLength);
97
 
92
 
98
     const options = {
93
     const options = {
99
-        ...wasmCheck.feature.simd ? segmentationDimensions.model144 : segmentationDimensions.model96,
94
+        ...segmentationDimensions.modelLandscape,
100
         virtualBackground
95
         virtualBackground
101
     };
96
     };
102
 
97
 

+ 1
- 13
react/features/stream-effects/virtual-background/vendor/README.md 查看文件

1
 # Virtual Background on stream effects
1
 # Virtual Background on stream effects
2
 
2
 
3
-> Inspired from https://ai.googleblog.com/2020/10/background-features-in-google-meet.html and https://github.com/Volcomix/virtual-background.git
3
+> From https://google.github.io/mediapipe/solutions/models.html#selfie-segmentation
4
 
4
 
5
 #### Canvas 2D + CPU
5
 #### Canvas 2D + CPU
6
 
6
 
22
 - [WebAssembly](https://webassembly.org/)
22
 - [WebAssembly](https://webassembly.org/)
23
 - [WebAssembly SIMD](https://github.com/WebAssembly/simd)
23
 - [WebAssembly SIMD](https://github.com/WebAssembly/simd)
24
 - [TFLite](https://blog.tensorflow.org/2020/07/accelerating-tensorflow-lite-xnnpack-integration.html)
24
 - [TFLite](https://blog.tensorflow.org/2020/07/accelerating-tensorflow-lite-xnnpack-integration.html)
25
-
26
-## LICENSE
27
-
28
-The mdoels vendored here were downloaded early January (they were available as early as the 4th), before Google switched the license away from Apache 2. Thus we understand they are not covered by the new license which according to the [model card](https://drive.google.com/file/d/1lnP1bRi9CSqQQXUHa13159vLELYDgDu0/view) dates from the 21st of January.
29
-
30
-We are not lawyers so do get legal advise if in doubt.
31
-
32
-References:
33
-
34
-- Model license discussion: https://github.com/tensorflow/tfjs/issues/4177
35
-- Current vendored model is discovered: https://github.com/tensorflow/tfjs/issues/4177#issuecomment-753934631
36
-- License change is noticed: https://github.com/tensorflow/tfjs/issues/4177#issuecomment-771536641

二进制
react/features/stream-effects/virtual-background/vendor/models/segm_full_v679.tflite 查看文件


二进制
react/features/stream-effects/virtual-background/vendor/models/segm_lite_v681.tflite 查看文件


二进制
react/features/stream-effects/virtual-background/vendor/models/selfie_segmentation_landscape.tflite 查看文件


正在加载...
取消
保存