Fix the WebGL memory leak for Blur effect in Chrome 77 and up (#4652)

package-lock.json

@@ -4007,37 +4007,36 @@
       }
     },
     "@tensorflow-models/body-pix": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-1.1.1.tgz",
-      "integrity": "sha512-l9bd+b3QI7OzJjw/OuhEfeGRb5l2lRivgDHGMvQbT2Snn8nV7odHSRW55NzhU7Khl7vga00TWo5QDuVnkevQmQ=="
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-1.1.2.tgz",
+      "integrity": "sha512-moCCTlP77v20HMg1e/Hs1LehCDLAKS32e6OUeI1MA/4HrRRO1Dq9engVCLFZUMO2+mJXdQeBdzexcFg0WQox7w=="
     },
     "@tensorflow/tfjs": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.2.2.tgz",
-      "integrity": "sha512-HfhSzL2eTWhlT0r/A5wmo+u3bHe+an16p5wsnFH3ujn21fQ8QtGpSfDHQZjWx1kVFaQnV6KBG+17MOrRHoHlLA==",
+      "version": "1.2.9",
+      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.2.9.tgz",
+      "integrity": "sha512-9UAQnSp638FyM5eedYEM+j2R7VcNajiFmkeT5EXtf7YIurmMFNEm1sbajKJx7/ckz31YcYrVoUPc/iLhhDQl2A==",
       "requires": {
-        "@tensorflow/tfjs-converter": "1.2.2",
-        "@tensorflow/tfjs-core": "1.2.2",
-        "@tensorflow/tfjs-data": "1.2.2",
-        "@tensorflow/tfjs-layers": "1.2.2"
+        "@tensorflow/tfjs-converter": "1.2.9",
+        "@tensorflow/tfjs-core": "1.2.9",
+        "@tensorflow/tfjs-data": "1.2.9",
+        "@tensorflow/tfjs-layers": "1.2.9"
       }
     },
     "@tensorflow/tfjs-converter": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.2.2.tgz",
-      "integrity": "sha512-NM2NcPRHpCNeJdBxHcYpmW9ZHTQ2lJFJgmgGpQ8CxSC9CtQB05bFONs3SKcwMNDE/69QBRVom5DYqLCVUg+A+g=="
+      "version": "1.2.9",
+      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.2.9.tgz",
+      "integrity": "sha512-OKmiuZicIgadT3Bv9BvM+oom7wRz9eC5rTglQnuv7VN9H0syFVuhf5oD1Ff70tGDhJjJgL+cPz01fZRxTXjRWA=="
     },
     "@tensorflow/tfjs-core": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.2.2.tgz",
-      "integrity": "sha512-2hCHMKjh3UNpLEjbAEaurrTGJyj/KpLtMSAraWgHA1vGY0kmk50BBSbgCDmXWUVm7lyh/SkCq4/GrGDZktEs3g==",
+      "version": "1.2.9",
+      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.2.9.tgz",
+      "integrity": "sha512-s0hHZSx6rGTlkkB8u8gs5n7sIPv1GXDNHmISRy+kqGzmlpkfI2kr6WXqOWQy6wFgjzopRD8cJQjBZ9USPZnYTQ==",
       "requires": {
         "@types/offscreencanvas": "~2019.3.0",
         "@types/seedrandom": "2.4.27",
         "@types/webgl-ext": "0.0.30",
         "@types/webgl2": "0.0.4",
         "node-fetch": "~2.1.2",
-        "rollup-plugin-visualizer": "~1.1.1",
         "seedrandom": "2.4.3"
       },
       "dependencies": {
@@ -4049,9 +4048,9 @@
       }
     },
     "@tensorflow/tfjs-data": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.2.2.tgz",
-      "integrity": "sha512-oHGBoGdnCl2RyouLKplQqo+iil0iJgPbi/aoHizhpO77UBuJXlKMblH8w5GbxVAw3hKxWlqzYpxPo6rVRgehNA==",
+      "version": "1.2.9",
+      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.2.9.tgz",
+      "integrity": "sha512-Ti9Cj3pte9butuEsK5OPq0Lcqdi4wVUdtQXm0o7iYOZ0umseRzfbIb6zbdqucc2MQzOMTnRoxN+FL7LZmncsHg==",
       "requires": {
         "@types/node-fetch": "^2.1.2",
         "node-fetch": "~2.1.2"
@@ -4065,9 +4064,9 @@
       }
     },
     "@tensorflow/tfjs-layers": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.2.2.tgz",
-      "integrity": "sha512-yzWZaZrCVpEyTkSrzMe4OOP4aGUfaaROE/zR9fPsPGGF8wLlbLNZUJjeYUmjy3G3pXGaM0mQUbLR5Vd707CVtQ=="
+      "version": "1.2.9",
+      "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.2.9.tgz",
+      "integrity": "sha512-OlXYaIb1rCk5dYmpaNsPEkO7R+T0oxfS3vQGIztNJB+YxrN8mwCu3hqgpbdKhAITiP+jxO0o+7bny8MsOCkOSQ=="
     },
     "@types/istanbul-lib-coverage": {
       "version": "2.0.1",
@@ -4092,14 +4091,14 @@
       }
     },
     "@types/node": {
-      "version": "12.0.10",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
-      "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ=="
+      "version": "12.7.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.5.tgz",
+      "integrity": "sha512-9fq4jZVhPNW8r+UYKnxF1e2HkDWOWKM5bC2/7c9wPV835I0aOrVbS/Hw/pWPk2uKrNXQqg9Z959Kz+IYDd5p3w=="
     },
     "@types/node-fetch": {
-      "version": "2.3.7",
-      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.3.7.tgz",
-      "integrity": "sha512-+bKtuxhj/TYSSP1r4CZhfmyA0vm/aDRQNo7vbAgf6/cZajn0SAniGGST07yvI4Q+q169WTa2/x9gEHfJrkcALw==",
+      "version": "2.5.2",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.2.tgz",
+      "integrity": "sha512-djYYKmdNRSBtL1x4CiE9UJb9yZhwtI1VC+UxZD0psNznrUj80ywsxKlEGAE+QL1qvLjPbfb24VosjkYM6W4RSQ==",
       "requires": {
         "@types/node": "*"
       }
@@ -16442,35 +16441,6 @@
         "inherits": "^2.0.1"
       }
     },
-    "rollup-plugin-visualizer": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/rollup-plugin-visualizer/-/rollup-plugin-visualizer-1.1.1.tgz",
-      "integrity": "sha512-7xkSKp+dyJmSC7jg2LXqViaHuOnF1VvIFCnsZEKjrgT5ZVyiLLSbeszxFcQSfNJILphqgAEmWAUz0Z4xYScrRw==",
-      "optional": true,
-      "requires": {
-        "mkdirp": "^0.5.1",
-        "opn": "^5.4.0",
-        "source-map": "^0.7.3",
-        "typeface-oswald": "0.0.54"
-      },
-      "dependencies": {
-        "opn": {
-          "version": "5.5.0",
-          "resolved": "https://registry.npmjs.org/opn/-/opn-5.5.0.tgz",
-          "integrity": "sha512-PqHpggC9bLV0VeWcdKhkpxY+3JTzetLSqTCWL/z/tFIbI6G8JCjondXklT1JinczLz2Xib62sSp0T/gKT4KksA==",
-          "optional": true,
-          "requires": {
-            "is-wsl": "^1.1.0"
-          }
-        },
-        "source-map": {
-          "version": "0.7.3",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz",
-          "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==",
-          "optional": true
-        }
-      }
-    },
     "rsvp": {
       "version": "4.8.5",
       "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
@@ -18460,12 +18430,6 @@
       "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
       "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
     },
-    "typeface-oswald": {
-      "version": "0.0.54",
-      "resolved": "https://registry.npmjs.org/typeface-oswald/-/typeface-oswald-0.0.54.tgz",
-      "integrity": "sha512-U1WMNp4qfy4/3khIfHMVAIKnNu941MXUfs3+H9R8PFgnoz42Hh9pboSFztWr86zut0eXC8byalmVhfkiKON/8Q==",
-      "optional": true
-    },
     "ua-parser-js": {
       "version": "0.7.17",
       "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.17.tgz",

package.json

@@ -37,8 +37,8 @@
     "@react-native-community/async-storage": "1.3.4",
     "@react-native-community/netinfo": "4.1.5",
     "@svgr/webpack": "4.3.2",
-    "@tensorflow-models/body-pix": "^1.0.1",
-    "@tensorflow/tfjs": "^1.1.2",
+    "@tensorflow-models/body-pix": "1.1.2",
+    "@tensorflow/tfjs": "1.2.9",
     "@webcomponents/url": "0.7.1",
     "amplitude-js": "4.5.2",
     "bc-css-flags": "3.0.0",

react/features/stream-effects/blur/JitsiStreamBlurEffect.js

@@ -100,7 +100,7 @@ export default class JitsiStreamBlurEffect {
         });
         this._maskFrameTimerWorker.postMessage({
             id: SET_INTERVAL,
-            timeMs: 200
+            timeMs: 50
         });
 
         return this._outputCanvasElement.captureStream(this._frameRate);
@@ -136,6 +136,11 @@ export default class JitsiStreamBlurEffect {
                 7, // Constant for background blur, integer values between 0-20
                 7 // Constant for edge blur, integer values between 0-20
             );
+
+            // Make sure we clear this buffer before feeding the segmentation data
+            // to drawBokehEffect for creating the blur. This fixes the memory leak
+            // that started happening in WebGL in Chrome 77 and up.
+            this._segmentationData = null;
         }
     }
 

react/features/stream-effects/blur/index.js

@@ -1,7 +1,7 @@
 // @flow
 
 import { load } from '@tensorflow-models/body-pix';
-
+import * as tfc from '@tensorflow/tfjs-core';
 import JitsiStreamBlurEffect from './JitsiStreamBlurEffect';
 
 /**
@@ -11,6 +11,12 @@ import JitsiStreamBlurEffect from './JitsiStreamBlurEffect';
  */
 const bpModelPromise = load(0.25);
 
+/**
+ * Configure the Tensor Flow model to use the webgl backend which is the
+ * most powerful backend for the browser.
+ */
+const webGlBackend = 'webgl';
+
 /**
  * Creates a new instance of JitsiStreamBlurEffect.
  *
@@ -21,5 +27,18 @@ export function createBlurEffect() {
         return Promise.reject(new Error('JitsiStreamBlurEffect not supported!'));
     }
 
-    return bpModelPromise.then(bpmodel => new JitsiStreamBlurEffect(bpmodel));
+    const setBackendPromise = new Promise((resolve, reject) => {
+        if (tfc.getBackend() === webGlBackend) {
+            resolve();
+
+            return;
+        }
+
+        return tfc.setBackend(webGlBackend)
+            .then(resolve, reject);
+    });
+
+    return setBackendPromise
+        .then(() => bpModelPromise)
+        .then(bpmodel => new JitsiStreamBlurEffect(bpmodel));
 }