Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

RTCUtils.js 69KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969
  1. /* global $,
  2. __filename,
  3. attachMediaStream,
  4. MediaStreamTrack,
  5. RTCIceCandidate: true,
  6. RTCPeerConnection,
  7. RTCSessionDescription: true,
  8. webkitMediaStream,
  9. webkitRTCPeerConnection,
  10. webkitURL
  11. */
  12. import { AVAILABLE_DEVICE } from '../../service/statistics/AnalyticsEvents';
  13. import CameraFacingMode from '../../service/RTC/CameraFacingMode';
  14. import EventEmitter from 'events';
  15. import { getLogger } from 'jitsi-meet-logger';
  16. import GlobalOnErrorHandler from '../util/GlobalOnErrorHandler';
  17. import JitsiTrackError from '../../JitsiTrackError';
  18. import Listenable from '../util/Listenable';
  19. import * as MediaType from '../../service/RTC/MediaType';
  20. import Resolutions from '../../service/RTC/Resolutions';
  21. import browser from '../browser';
  22. import RTCEvents from '../../service/RTC/RTCEvents';
  23. import ortcRTCPeerConnection from './ortc/RTCPeerConnection';
  24. import screenObtainer from './ScreenObtainer';
  25. import SDPUtil from '../xmpp/SDPUtil';
  26. import Statistics from '../statistics/statistics';
  27. import VideoType from '../../service/RTC/VideoType';
  28. const logger = getLogger(__filename);
  29. // XXX Don't require Temasys unless it's to be used because it doesn't run on
  30. // React Native, for example.
  31. const AdapterJS
  32. = browser.isTemasysPluginUsed()
  33. ? require('./adapter.screenshare')
  34. : undefined;
  35. // Require adapter only for certain browsers. This is being done for
  36. // react-native, which has its own shims, and while browsers are being migrated
  37. // over to use adapter's shims.
  38. if (browser.usesNewGumFlow()) {
  39. require('webrtc-adapter');
  40. }
  41. const eventEmitter = new EventEmitter();
  42. const AVAILABLE_DEVICES_POLL_INTERVAL_TIME = 3000; // ms
  43. /**
  44. * Default resolution to obtain for video tracks if no resolution is specified.
  45. * This default is used for old gum flow only, as new gum flow uses
  46. * {@link DEFAULT_CONSTRAINTS}.
  47. */
  48. const OLD_GUM_DEFAULT_RESOLUTION = 720;
  49. /**
  50. * Default devices to obtain when no specific devices are specified. This
  51. * default is used for old gum flow only.
  52. */
  53. const OLD_GUM_DEFAULT_DEVICES = [ 'audio', 'video' ];
  54. /**
  55. * Default MediaStreamConstraints to use for calls to getUserMedia.
  56. *
  57. * @private
  58. */
  59. const DEFAULT_CONSTRAINTS = {
  60. video: {
  61. aspectRatio: 16 / 9,
  62. height: {
  63. ideal: 1080,
  64. max: 1080,
  65. min: 240
  66. }
  67. }
  68. };
  69. // TODO (brian): Move this devices hash, maybe to a model, so RTCUtils remains
  70. // stateless.
  71. const devices = {
  72. audio: false,
  73. video: false
  74. };
  75. // Currently audio output device change is supported only in Chrome and
  76. // default output always has 'default' device ID
  77. let audioOutputDeviceId = 'default'; // default device
  78. // whether user has explicitly set a device to use
  79. let audioOutputChanged = false;
  80. // Disables all audio processing
  81. let disableAP = false;
  82. // Disables Acoustic Echo Cancellation
  83. let disableAEC = false;
  84. // Disables Noise Suppression
  85. let disableNS = false;
  86. // Disables Automatic Gain Control
  87. let disableAGC = false;
  88. // Disables Highpass Filter
  89. let disableHPF = false;
  90. const featureDetectionAudioEl = document.createElement('audio');
  91. const isAudioOutputDeviceChangeAvailable
  92. = typeof featureDetectionAudioEl.setSinkId !== 'undefined';
  93. let currentlyAvailableMediaDevices;
  94. /**
  95. * "rawEnumerateDevicesWithCallback" will be initialized only after WebRTC is
  96. * ready. Otherwise it is too early to assume that the devices listing is not
  97. * supported.
  98. */
  99. let rawEnumerateDevicesWithCallback;
  100. /**
  101. *
  102. */
  103. function initRawEnumerateDevicesWithCallback() {
  104. rawEnumerateDevicesWithCallback
  105. = navigator.mediaDevices && navigator.mediaDevices.enumerateDevices
  106. ? function(callback) {
  107. navigator.mediaDevices.enumerateDevices().then(
  108. callback,
  109. () => callback([]));
  110. }
  111. // Safari:
  112. // "ReferenceError: Can't find variable: MediaStreamTrack" when
  113. // Temasys plugin is not installed yet, have to delay this call
  114. // until WebRTC is ready.
  115. : MediaStreamTrack && MediaStreamTrack.getSources
  116. ? function(callback) {
  117. MediaStreamTrack.getSources(
  118. sources =>
  119. callback(
  120. sources.map(convertMediaStreamTrackSource)));
  121. }
  122. : undefined;
  123. }
  124. // TODO: currently no browser supports 'devicechange' event even in nightly
  125. // builds so no feature/browser detection is used at all. However in future this
  126. // should be changed to some expression. Progress on 'devicechange' event
  127. // implementation for Chrome/Opera/NWJS can be tracked at
  128. // https://bugs.chromium.org/p/chromium/issues/detail?id=388648, for Firefox -
  129. // at https://bugzilla.mozilla.org/show_bug.cgi?id=1152383. More information on
  130. // 'devicechange' event can be found in spec -
  131. // http://w3c.github.io/mediacapture-main/#event-mediadevices-devicechange
  132. // TODO: check MS Edge
  133. const isDeviceChangeEventSupported = false;
  134. let rtcReady = false;
  135. /**
  136. *
  137. * @param constraints
  138. * @param isNewStyleConstraintsSupported
  139. * @param resolution
  140. */
  141. function setResolutionConstraints(
  142. constraints,
  143. isNewStyleConstraintsSupported,
  144. resolution) {
  145. if (Resolutions[resolution]) {
  146. if (isNewStyleConstraintsSupported) {
  147. constraints.video.width = {
  148. ideal: Resolutions[resolution].width
  149. };
  150. constraints.video.height = {
  151. ideal: Resolutions[resolution].height
  152. };
  153. }
  154. constraints.video.mandatory.minWidth = Resolutions[resolution].width;
  155. constraints.video.mandatory.minHeight = Resolutions[resolution].height;
  156. }
  157. if (constraints.video.mandatory.minWidth) {
  158. constraints.video.mandatory.maxWidth
  159. = constraints.video.mandatory.minWidth;
  160. }
  161. if (constraints.video.mandatory.minHeight) {
  162. constraints.video.mandatory.maxHeight
  163. = constraints.video.mandatory.minHeight;
  164. }
  165. }
  166. /**
  167. * @param {string[]} um required user media types
  168. *
  169. * @param {Object} [options={}] optional parameters
  170. * @param {string} options.resolution
  171. * @param {number} options.bandwidth
  172. * @param {number} options.fps
  173. * @param {string} options.desktopStream
  174. * @param {string} options.cameraDeviceId
  175. * @param {string} options.micDeviceId
  176. * @param {CameraFacingMode} options.facingMode
  177. * @param {bool} firefox_fake_device
  178. */
  179. function getConstraints(um, options) {
  180. const constraints = { audio: false,
  181. video: false };
  182. // Don't mix new and old style settings for Chromium as this leads
  183. // to TypeError in new Chromium versions. @see
  184. // https://bugs.chromium.org/p/chromium/issues/detail?id=614716
  185. // This is a temporary solution, in future we will fully split old and
  186. // new style constraints when new versions of Chromium and Firefox will
  187. // have stable support of new constraints format. For more information
  188. // @see https://github.com/jitsi/lib-jitsi-meet/pull/136
  189. const isNewStyleConstraintsSupported
  190. = browser.isFirefox()
  191. || browser.isEdge()
  192. || browser.isReactNative()
  193. || browser.isTemasysPluginUsed();
  194. if (um.indexOf('video') >= 0) {
  195. // same behaviour as true
  196. constraints.video = { mandatory: {},
  197. optional: [] };
  198. if (options.cameraDeviceId) {
  199. if (isNewStyleConstraintsSupported) {
  200. // New style of setting device id.
  201. constraints.video.deviceId = options.cameraDeviceId;
  202. }
  203. // Old style.
  204. constraints.video.optional.push({
  205. sourceId: options.cameraDeviceId
  206. });
  207. } else {
  208. // Prefer the front i.e. user-facing camera (to the back i.e.
  209. // environment-facing camera, for example).
  210. // TODO: Maybe use "exact" syntax if options.facingMode is defined,
  211. // but this probably needs to be decided when updating other
  212. // constraints, as we currently don't use "exact" syntax anywhere.
  213. const facingMode = options.facingMode || CameraFacingMode.USER;
  214. if (isNewStyleConstraintsSupported) {
  215. constraints.video.facingMode = facingMode;
  216. }
  217. constraints.video.optional.push({
  218. facingMode
  219. });
  220. }
  221. if (options.minFps || options.maxFps || options.fps) {
  222. // for some cameras it might be necessary to request 30fps
  223. // so they choose 30fps mjpg over 10fps yuy2
  224. if (options.minFps || options.fps) {
  225. // Fall back to options.fps for backwards compatibility
  226. options.minFps = options.minFps || options.fps;
  227. constraints.video.mandatory.minFrameRate = options.minFps;
  228. }
  229. if (options.maxFps) {
  230. constraints.video.mandatory.maxFrameRate = options.maxFps;
  231. }
  232. }
  233. setResolutionConstraints(
  234. constraints, isNewStyleConstraintsSupported, options.resolution);
  235. }
  236. if (um.indexOf('audio') >= 0) {
  237. if (browser.isReactNative()) {
  238. // The react-native-webrtc project that we're currently using
  239. // expects the audio constraint to be a boolean.
  240. constraints.audio = true;
  241. } else if (browser.isFirefox()) {
  242. if (options.micDeviceId) {
  243. constraints.audio = {
  244. mandatory: {},
  245. deviceId: options.micDeviceId, // new style
  246. optional: [ {
  247. sourceId: options.micDeviceId // old style
  248. } ] };
  249. } else {
  250. constraints.audio = true;
  251. }
  252. } else {
  253. // same behaviour as true
  254. constraints.audio = { mandatory: {},
  255. optional: [] };
  256. if (options.micDeviceId) {
  257. if (isNewStyleConstraintsSupported) {
  258. // New style of setting device id.
  259. constraints.audio.deviceId = options.micDeviceId;
  260. }
  261. // Old style.
  262. constraints.audio.optional.push({
  263. sourceId: options.micDeviceId
  264. });
  265. }
  266. // if it is good enough for hangouts...
  267. constraints.audio.optional.push(
  268. { echoCancellation: !disableAEC && !disableAP },
  269. { googEchoCancellation: !disableAEC && !disableAP },
  270. { googAutoGainControl: !disableAGC && !disableAP },
  271. { googNoiseSuppression: !disableNS && !disableAP },
  272. { googHighpassFilter: !disableHPF && !disableAP },
  273. { googNoiseSuppression2: !disableNS && !disableAP },
  274. { googEchoCancellation2: !disableAEC && !disableAP },
  275. { googAutoGainControl2: !disableAGC && !disableAP }
  276. );
  277. }
  278. }
  279. if (um.indexOf('screen') >= 0) {
  280. if (browser.isChrome()) {
  281. constraints.video = {
  282. mandatory: {
  283. chromeMediaSource: 'screen',
  284. maxWidth: window.screen.width,
  285. maxHeight: window.screen.height,
  286. maxFrameRate: 3
  287. },
  288. optional: []
  289. };
  290. } else if (browser.isTemasysPluginUsed()) {
  291. constraints.video = {
  292. optional: [
  293. {
  294. sourceId: AdapterJS.WebRTCPlugin.plugin.screensharingKey
  295. }
  296. ]
  297. };
  298. } else if (browser.isFirefox()) {
  299. constraints.video = {
  300. mozMediaSource: 'window',
  301. mediaSource: 'window'
  302. };
  303. } else {
  304. const errmsg
  305. = '\'screen\' WebRTC media source is supported only in Chrome'
  306. + ' and with Temasys plugin';
  307. GlobalOnErrorHandler.callErrorHandler(new Error(errmsg));
  308. logger.error(errmsg);
  309. }
  310. }
  311. if (um.indexOf('desktop') >= 0) {
  312. constraints.video = {
  313. mandatory: {
  314. chromeMediaSource: 'desktop',
  315. chromeMediaSourceId: options.desktopStream,
  316. maxWidth: window.screen.width,
  317. maxHeight: window.screen.height,
  318. maxFrameRate: 3
  319. },
  320. optional: []
  321. };
  322. }
  323. if (options.bandwidth) {
  324. if (!constraints.video) {
  325. // same behaviour as true
  326. constraints.video = { mandatory: {},
  327. optional: [] };
  328. }
  329. constraints.video.optional.push({ bandwidth: options.bandwidth });
  330. }
  331. // we turn audio for both audio and video tracks, the fake audio & video
  332. // seems to work only when enabled in one getUserMedia call, we cannot get
  333. // fake audio separate by fake video this later can be a problem with some
  334. // of the tests
  335. if (browser.isFirefox() && options.firefox_fake_device) {
  336. // seems to be fixed now, removing this experimental fix, as having
  337. // multiple audio tracks brake the tests
  338. // constraints.audio = true;
  339. constraints.fake = true;
  340. }
  341. return constraints;
  342. }
  343. /**
  344. * Creates a constraints object to be passed into a call to getUserMedia.
  345. *
  346. * @param {Array} um - An array of user media types to get. The accepted
  347. * types are "video", "audio", and "desktop."
  348. * @param {Object} options - Various values to be added to the constraints.
  349. * @param {string} options.cameraDeviceId - The device id for the video
  350. * capture device to get video from.
  351. * @param {Object} options.constraints - Default constraints object to use
  352. * as a base for the returned constraints.
  353. * @param {Object} options.desktopStream - The desktop source id from which
  354. * to capture a desktop sharing video.
  355. * @param {string} options.facingMode - Which direction the camera is
  356. * pointing to.
  357. * @param {string} options.micDeviceId - The device id for the audio capture
  358. * device to get audio from.
  359. * @private
  360. * @returns {Object}
  361. */
  362. function newGetConstraints(um = [], options = {}) {
  363. // Create a deep copy of the constraints to avoid any modification of
  364. // the passed in constraints object.
  365. const constraints = JSON.parse(JSON.stringify(
  366. options.constraints || DEFAULT_CONSTRAINTS));
  367. if (um.indexOf('video') >= 0) {
  368. if (!constraints.video) {
  369. constraints.video = {};
  370. }
  371. if (options.cameraDeviceId) {
  372. constraints.video.deviceId = options.cameraDeviceId;
  373. } else {
  374. const facingMode = options.facingMode || CameraFacingMode.USER;
  375. constraints.video.facingMode = facingMode;
  376. }
  377. } else {
  378. constraints.video = false;
  379. }
  380. if (um.indexOf('audio') >= 0) {
  381. if (!constraints.audio || typeof constraints.audio === 'boolean') {
  382. constraints.audio = {};
  383. }
  384. // NOTE(brian): the new-style ('advanced' instead of 'optional')
  385. // doesn't seem to carry through the googXXX constraints
  386. // Changing back to 'optional' here (even with video using
  387. // the 'advanced' style) allows them to be passed through
  388. // but also requires the device id to capture to be set in optional
  389. // as sourceId otherwise the constraints are considered malformed.
  390. if (!constraints.audio.optional) {
  391. constraints.audio.optional = [];
  392. }
  393. constraints.audio.optional.push(
  394. { sourceId: options.micDeviceId },
  395. { echoCancellation: !disableAEC && !disableAP },
  396. { googEchoCancellation: !disableAEC && !disableAP },
  397. { googAutoGainControl: !disableAGC && !disableAP },
  398. { googNoiseSuppression: !disableNS && !disableAP },
  399. { googHighpassFilter: !disableHPF && !disableAP },
  400. { googNoiseSuppression2: !disableNS && !disableAP },
  401. { googEchoCancellation2: !disableAEC && !disableAP },
  402. { googAutoGainControl2: !disableAGC && !disableAP }
  403. );
  404. } else {
  405. constraints.audio = false;
  406. }
  407. if (um.indexOf('desktop') >= 0) {
  408. if (!constraints.video || typeof constraints.video === 'boolean') {
  409. constraints.video = {};
  410. }
  411. constraints.video = {
  412. mandatory: {
  413. chromeMediaSource: 'desktop',
  414. chromeMediaSourceId: options.desktopStream,
  415. maxWidth: window.screen.width,
  416. maxHeight: window.screen.height,
  417. maxFrameRate: 3
  418. }
  419. };
  420. }
  421. return constraints;
  422. }
  423. /**
  424. * Sets the availbale devices based on the options we requested and the
  425. * streams we received.
  426. * @param um the options we requested to getUserMedia.
  427. * @param stream the stream we received from calling getUserMedia.
  428. */
  429. function setAvailableDevices(um, stream) {
  430. const audioTracksReceived = stream && stream.getAudioTracks().length > 0;
  431. const videoTracksReceived = stream && stream.getVideoTracks().length > 0;
  432. if (um.indexOf('video') !== -1) {
  433. devices.video = videoTracksReceived;
  434. }
  435. if (um.indexOf('audio') !== -1) {
  436. devices.audio = audioTracksReceived;
  437. }
  438. eventEmitter.emit(RTCEvents.AVAILABLE_DEVICES_CHANGED, devices);
  439. }
  440. /**
  441. * Checks if new list of available media devices differs from previous one.
  442. * @param {MediaDeviceInfo[]} newDevices - list of new devices.
  443. * @returns {boolean} - true if list is different, false otherwise.
  444. */
  445. function compareAvailableMediaDevices(newDevices) {
  446. if (newDevices.length !== currentlyAvailableMediaDevices.length) {
  447. return true;
  448. }
  449. /* eslint-disable newline-per-chained-call */
  450. return (
  451. newDevices.map(mediaDeviceInfoToJSON).sort().join('')
  452. !== currentlyAvailableMediaDevices
  453. .map(mediaDeviceInfoToJSON).sort().join(''));
  454. /* eslint-enable newline-per-chained-call */
  455. /**
  456. *
  457. * @param info
  458. */
  459. function mediaDeviceInfoToJSON(info) {
  460. return JSON.stringify({
  461. kind: info.kind,
  462. deviceId: info.deviceId,
  463. groupId: info.groupId,
  464. label: info.label,
  465. facing: info.facing
  466. });
  467. }
  468. }
  469. /**
  470. * Periodically polls enumerateDevices() method to check if list of media
  471. * devices has changed. This is temporary workaround until 'devicechange' event
  472. * will be supported by browsers.
  473. */
  474. function pollForAvailableMediaDevices() {
  475. // Here we use plain navigator.mediaDevices.enumerateDevices instead of
  476. // wrapped because we just need to know the fact the devices changed, labels
  477. // do not matter. This fixes situation when we have no devices initially,
  478. // and then plug in a new one.
  479. if (rawEnumerateDevicesWithCallback) {
  480. rawEnumerateDevicesWithCallback(ds => {
  481. // We don't fire RTCEvents.DEVICE_LIST_CHANGED for the first time
  482. // we call enumerateDevices(). This is the initial step.
  483. if (typeof currentlyAvailableMediaDevices === 'undefined') {
  484. currentlyAvailableMediaDevices = ds.slice(0);
  485. } else if (compareAvailableMediaDevices(ds)) {
  486. onMediaDevicesListChanged(ds);
  487. }
  488. window.setTimeout(pollForAvailableMediaDevices,
  489. AVAILABLE_DEVICES_POLL_INTERVAL_TIME);
  490. });
  491. }
  492. }
  493. /**
  494. * Sends analytics event with the passed device list.
  495. *
  496. * @param {Array<MediaDeviceInfo>} deviceList - List with info about the
  497. * available devices.
  498. * @returns {void}
  499. */
  500. function sendDeviceListToAnalytics(deviceList) {
  501. const audioInputDeviceCount
  502. = deviceList.filter(d => d.kind === 'audioinput').length;
  503. const audioOutputDeviceCount
  504. = deviceList.filter(d => d.kind === 'audiooutput').length;
  505. const videoInputDeviceCount
  506. = deviceList.filter(d => d.kind === 'videoinput').length;
  507. const videoOutputDeviceCount
  508. = deviceList.filter(d => d.kind === 'videooutput').length;
  509. deviceList.forEach(device => {
  510. const attributes = {
  511. 'audio_input_device_count': audioInputDeviceCount,
  512. 'audio_output_device_count': audioOutputDeviceCount,
  513. 'video_input_device_count': videoInputDeviceCount,
  514. 'video_output_device_count': videoOutputDeviceCount,
  515. 'device_id': device.deviceId,
  516. 'device_group_id': device.groupId,
  517. 'device_kind': device.kind,
  518. 'device_label': device.label
  519. };
  520. Statistics.sendAnalytics(AVAILABLE_DEVICE, attributes);
  521. });
  522. }
  523. /**
  524. * Event handler for the 'devicechange' event.
  525. *
  526. * @param {MediaDeviceInfo[]} devices - list of media devices.
  527. * @emits RTCEvents.DEVICE_LIST_CHANGED
  528. */
  529. function onMediaDevicesListChanged(devicesReceived) {
  530. currentlyAvailableMediaDevices = devicesReceived.slice(0);
  531. logger.info(
  532. 'list of media devices has changed:',
  533. currentlyAvailableMediaDevices);
  534. sendDeviceListToAnalytics(currentlyAvailableMediaDevices);
  535. const videoInputDevices
  536. = currentlyAvailableMediaDevices.filter(d => d.kind === 'videoinput');
  537. const audioInputDevices
  538. = currentlyAvailableMediaDevices.filter(d => d.kind === 'audioinput');
  539. const videoInputDevicesWithEmptyLabels
  540. = videoInputDevices.filter(d => d.label === '');
  541. const audioInputDevicesWithEmptyLabels
  542. = audioInputDevices.filter(d => d.label === '');
  543. if (videoInputDevices.length
  544. && videoInputDevices.length
  545. === videoInputDevicesWithEmptyLabels.length) {
  546. devices.video = false;
  547. }
  548. if (audioInputDevices.length
  549. && audioInputDevices.length
  550. === audioInputDevicesWithEmptyLabels.length) {
  551. devices.audio = false;
  552. }
  553. eventEmitter.emit(RTCEvents.DEVICE_LIST_CHANGED, devicesReceived);
  554. }
  555. /**
  556. * Apply function with arguments if function exists.
  557. * Do nothing if function not provided.
  558. * @param {function} [fn] function to apply
  559. * @param {Array} [args=[]] arguments for function
  560. */
  561. function maybeApply(fn, args) {
  562. fn && fn(...args);
  563. }
  564. /**
  565. * Wrap `getUserMedia` in order to convert between callback and Promise based
  566. * APIs.
  567. * @param {Function} getUserMedia native function
  568. * @returns {Function} wrapped function
  569. */
  570. function wrapGetUserMedia(getUserMedia, usePromises = false) {
  571. let gUM;
  572. if (usePromises) {
  573. gUM = function(constraints, successCallback, errorCallback) {
  574. return getUserMedia(constraints)
  575. .then(stream => {
  576. maybeApply(successCallback, [ stream ]);
  577. return stream;
  578. })
  579. .catch(error => {
  580. maybeApply(errorCallback, [ error ]);
  581. throw error;
  582. });
  583. };
  584. } else {
  585. gUM = function(constraints, successCallback, errorCallback) {
  586. getUserMedia(constraints, stream => {
  587. maybeApply(successCallback, [ stream ]);
  588. }, error => {
  589. maybeApply(errorCallback, [ error ]);
  590. });
  591. };
  592. }
  593. return gUM;
  594. }
  595. /**
  596. * Use old MediaStreamTrack to get devices list and
  597. * convert it to enumerateDevices format.
  598. * @param {Function} callback function to call when received devices list.
  599. */
  600. function enumerateDevicesThroughMediaStreamTrack(callback) {
  601. MediaStreamTrack.getSources(
  602. sources => callback(sources.map(convertMediaStreamTrackSource)));
  603. }
  604. /**
  605. * Converts MediaStreamTrack Source to enumerateDevices format.
  606. * @param {Object} source
  607. */
  608. function convertMediaStreamTrackSource(source) {
  609. const kind = (source.kind || '').toLowerCase();
  610. return {
  611. facing: source.facing || null,
  612. label: source.label,
  613. // theoretically deprecated MediaStreamTrack.getSources should
  614. // not return 'audiooutput' devices but let's handle it in any
  615. // case
  616. kind: kind
  617. ? kind === 'audiooutput' ? kind : `${kind}input`
  618. : null,
  619. deviceId: source.id,
  620. groupId: source.groupId || null
  621. };
  622. }
  623. /**
  624. * Handles the newly created Media Streams.
  625. * @param streams the new Media Streams
  626. * @param resolution the resolution of the video streams
  627. * @returns {*[]} object that describes the new streams
  628. */
  629. function handleLocalStream(streams, resolution) {
  630. let audioStream, desktopStream, videoStream;
  631. const res = [];
  632. // XXX The function obtainAudioAndVideoPermissions has examined the type of
  633. // the browser, its capabilities, etc. and has taken the decision whether to
  634. // invoke getUserMedia per device (e.g. Firefox) or once for both audio and
  635. // video (e.g. Chrome). In order to not duplicate the logic here, examine
  636. // the specified streams and figure out what we've received based on
  637. // obtainAudioAndVideoPermissions' decision.
  638. if (streams) {
  639. // As mentioned above, certian types of browser (e.g. Chrome) support
  640. // (with a result which meets our requirements expressed bellow) calling
  641. // getUserMedia once for both audio and video.
  642. const audioVideo = streams.audioVideo;
  643. if (audioVideo) {
  644. const NativeMediaStream
  645. = window.webkitMediaStream || window.MediaStream;
  646. const audioTracks = audioVideo.getAudioTracks();
  647. if (audioTracks.length) {
  648. // eslint-disable-next-line new-cap
  649. audioStream = new NativeMediaStream();
  650. for (let i = 0; i < audioTracks.length; i++) {
  651. audioStream.addTrack(audioTracks[i]);
  652. }
  653. }
  654. const videoTracks = audioVideo.getVideoTracks();
  655. if (videoTracks.length) {
  656. // eslint-disable-next-line new-cap
  657. videoStream = new NativeMediaStream();
  658. for (let j = 0; j < videoTracks.length; j++) {
  659. videoStream.addTrack(videoTracks[j]);
  660. }
  661. }
  662. } else {
  663. // On other types of browser (e.g. Firefox) we choose (namely,
  664. // obtainAudioAndVideoPermissions) to call getUserMedia per device
  665. // (type).
  666. audioStream = streams.audio;
  667. videoStream = streams.video;
  668. }
  669. desktopStream = streams.desktop;
  670. }
  671. if (desktopStream) {
  672. const { stream, sourceId, sourceType } = desktopStream;
  673. res.push({
  674. stream,
  675. sourceId,
  676. sourceType,
  677. track: stream.getVideoTracks()[0],
  678. mediaType: MediaType.VIDEO,
  679. videoType: VideoType.DESKTOP
  680. });
  681. }
  682. if (audioStream) {
  683. res.push({
  684. stream: audioStream,
  685. track: audioStream.getAudioTracks()[0],
  686. mediaType: MediaType.AUDIO,
  687. videoType: null
  688. });
  689. }
  690. if (videoStream) {
  691. res.push({
  692. stream: videoStream,
  693. track: videoStream.getVideoTracks()[0],
  694. mediaType: MediaType.VIDEO,
  695. videoType: VideoType.CAMERA,
  696. resolution
  697. });
  698. }
  699. return res;
  700. }
  701. /**
  702. * Represents a default implementation of setting a <tt>MediaStream</tt> as the
  703. * source of a video element that tries to be browser-agnostic through feature
  704. * checking. Note though that it was not completely clear from the predating
  705. * browser-specific implementations what &quot;videoSrc&quot; was because one
  706. * implementation of {@link RTCUtils#getVideoSrc} would return
  707. * <tt>MediaStream</tt> (e.g. Firefox), another a <tt>string</tt> representation
  708. * of the <tt>URL</tt> of the <tt>MediaStream</tt> (e.g. Chrome) and the return
  709. * value was only used by {@link RTCUIHelper#getVideoId} which itself did not
  710. * appear to be used anywhere. Generally, the implementation will try to follow
  711. * the related standards i.e. work with the <tt>srcObject</tt> and <tt>src</tt>
  712. * properties of the specified <tt>element</tt> taking into account vender
  713. * prefixes.
  714. *
  715. * @param element the element whose video source/src is to be set to the
  716. * specified <tt>stream</tt>
  717. * @param {MediaStream} stream the <tt>MediaStream</tt> to set as the video
  718. * source/src of <tt>element</tt>
  719. */
  720. function defaultSetVideoSrc(element, stream) {
  721. // srcObject
  722. let srcObjectPropertyName = 'srcObject';
  723. if (!(srcObjectPropertyName in element)) {
  724. srcObjectPropertyName = 'mozSrcObject';
  725. if (!(srcObjectPropertyName in element)) {
  726. srcObjectPropertyName = null;
  727. }
  728. }
  729. if (srcObjectPropertyName) {
  730. element[srcObjectPropertyName] = stream;
  731. return;
  732. }
  733. // src
  734. let src;
  735. if (stream) {
  736. src = stream.jitsiObjectURL;
  737. // Save the created URL for stream so we can reuse it and not keep
  738. // creating URLs.
  739. if (!src) {
  740. stream.jitsiObjectURL
  741. = src
  742. = (URL || webkitURL).createObjectURL(stream);
  743. }
  744. }
  745. element.src = src || '';
  746. }
  747. /**
  748. *
  749. */
  750. class RTCUtils extends Listenable {
  751. /**
  752. *
  753. */
  754. constructor() {
  755. super(eventEmitter);
  756. }
  757. /**
  758. * Depending on the browser, sets difference instance methods for
  759. * interacting with user media and adds methods to native webrtc related
  760. * objects. Also creates an instance variable for peer connection
  761. * constraints.
  762. *
  763. * @param {Object} options
  764. * @returns {void}
  765. */
  766. init(options = {}) {
  767. if (typeof options.disableAEC === 'boolean') {
  768. disableAEC = options.disableAEC;
  769. logger.info(`Disable AEC: ${disableAEC}`);
  770. }
  771. if (typeof options.disableNS === 'boolean') {
  772. disableNS = options.disableNS;
  773. logger.info(`Disable NS: ${disableNS}`);
  774. }
  775. if (typeof options.disableAP === 'boolean') {
  776. disableAP = options.disableAP;
  777. logger.info(`Disable AP: ${disableAP}`);
  778. }
  779. if (typeof options.disableAGC === 'boolean') {
  780. disableAGC = options.disableAGC;
  781. logger.info(`Disable AGC: ${disableAGC}`);
  782. }
  783. if (typeof options.disableHPF === 'boolean') {
  784. disableHPF = options.disableHPF;
  785. logger.info(`Disable HPF: ${disableHPF}`);
  786. }
  787. // Initialize rawEnumerateDevicesWithCallback
  788. initRawEnumerateDevicesWithCallback();
  789. return new Promise((resolve, reject) => {
  790. if (browser.usesNewGumFlow()) {
  791. this.RTCPeerConnectionType = window.RTCPeerConnection;
  792. this.getUserMedia
  793. = (constraints, successCallback, errorCallback) =>
  794. window.navigator.mediaDevices
  795. .getUserMedia(constraints)
  796. .then(stream => {
  797. successCallback && successCallback(stream);
  798. return stream;
  799. })
  800. .catch(err => {
  801. errorCallback && errorCallback(err);
  802. return Promise.reject(err);
  803. });
  804. this.enumerateDevices = callback =>
  805. window.navigator.mediaDevices.enumerateDevices()
  806. .then(foundDevices => {
  807. callback(foundDevices);
  808. return foundDevices;
  809. })
  810. .catch(err => {
  811. logger.error(`Error enumerating devices: ${err}`);
  812. callback([]);
  813. return [];
  814. });
  815. this.attachMediaStream
  816. = wrapAttachMediaStream((element, stream) => {
  817. if (element) {
  818. element.srcObject = stream;
  819. }
  820. return element;
  821. });
  822. this.getStreamID = stream => stream.id;
  823. this.getTrackID = track => track.id;
  824. } else if (browser.isChrome() // this is chrome < 61
  825. || browser.isOpera()
  826. || browser.isNWJS()
  827. || browser.isElectron()
  828. || browser.isReactNative()) {
  829. this.RTCPeerConnectionType = webkitRTCPeerConnection;
  830. const getUserMedia
  831. = navigator.webkitGetUserMedia.bind(navigator);
  832. this.getUserMedia = wrapGetUserMedia(getUserMedia);
  833. this.enumerateDevices = rawEnumerateDevicesWithCallback;
  834. this.attachMediaStream
  835. = wrapAttachMediaStream((element, stream) => {
  836. defaultSetVideoSrc(element, stream);
  837. return element;
  838. });
  839. this.getStreamID = function(stream) {
  840. // A. MediaStreams from FF endpoints have the characters '{'
  841. // and '}' that make jQuery choke.
  842. // B. The react-native-webrtc implementation that we use on
  843. // React Native at the time of this writing returns a number
  844. // for the id of MediaStream. Let's just say that a number
  845. // contains no special characters.
  846. const id = stream.id;
  847. // XXX The return statement is affected by automatic
  848. // semicolon insertion (ASI). No line terminator is allowed
  849. // between the return keyword and the expression.
  850. return (
  851. typeof id === 'number'
  852. ? id
  853. : SDPUtil.filterSpecialChars(id));
  854. };
  855. this.getTrackID = function(track) {
  856. return track.id;
  857. };
  858. if (!webkitMediaStream.prototype.getVideoTracks) {
  859. webkitMediaStream.prototype.getVideoTracks = function() {
  860. return this.videoTracks;
  861. };
  862. }
  863. if (!webkitMediaStream.prototype.getAudioTracks) {
  864. webkitMediaStream.prototype.getAudioTracks = function() {
  865. return this.audioTracks;
  866. };
  867. }
  868. } else if (browser.isEdge()) {
  869. this.RTCPeerConnectionType = ortcRTCPeerConnection;
  870. this.getUserMedia
  871. = wrapGetUserMedia(
  872. navigator.mediaDevices.getUserMedia.bind(
  873. navigator.mediaDevices),
  874. true);
  875. this.enumerateDevices = rawEnumerateDevicesWithCallback;
  876. this.attachMediaStream
  877. = wrapAttachMediaStream((element, stream) => {
  878. defaultSetVideoSrc(element, stream);
  879. return element;
  880. });
  881. // ORTC does not generate remote MediaStreams so those are
  882. // manually created by the ORTC shim. This means that their
  883. // id (internally generated) does not match the stream id
  884. // signaled into the remote SDP. Therefore, the shim adds a
  885. // custom jitsiRemoteId property with the original stream id.
  886. this.getStreamID = function(stream) {
  887. const id = stream.jitsiRemoteId || stream.id;
  888. return SDPUtil.filterSpecialChars(id);
  889. };
  890. // Remote MediaStreamTracks generated by ORTC (within a
  891. // RTCRtpReceiver) have an internally/random id which does not
  892. // match the track id signaled in the remote SDP. The shim adds
  893. // a custom jitsi-id property with the original track id.
  894. this.getTrackID = function(track) {
  895. return track.jitsiRemoteId || track.id;
  896. };
  897. } else if (browser.isTemasysPluginUsed()) {
  898. // Detect IE/Safari
  899. const webRTCReadyCb = () => {
  900. this.RTCPeerConnectionType = RTCPeerConnection;
  901. this.getUserMedia = window.getUserMedia;
  902. this.enumerateDevices
  903. = enumerateDevicesThroughMediaStreamTrack;
  904. this.attachMediaStream
  905. = wrapAttachMediaStream((element, stream) => {
  906. if (stream) {
  907. if (stream.id === 'dummyAudio'
  908. || stream.id === 'dummyVideo') {
  909. return;
  910. }
  911. // The container must be visible in order to
  912. // play or attach the stream when Temasys plugin
  913. // is in use
  914. const containerSel = $(element);
  915. if (browser.isTemasysPluginUsed()
  916. && !containerSel.is(':visible')) {
  917. containerSel.show();
  918. }
  919. const video
  920. = stream.getVideoTracks().length > 0;
  921. if (video && !$(element).is(':visible')) {
  922. throw new Error(
  923. 'video element must be visible to'
  924. + ' attach video stream');
  925. }
  926. }
  927. return attachMediaStream(element, stream);
  928. });
  929. this.getStreamID
  930. = stream => SDPUtil.filterSpecialChars(stream.label);
  931. this.getTrackID
  932. = track => track.id;
  933. onReady(
  934. options,
  935. this.getUserMediaWithConstraints.bind(this));
  936. };
  937. const webRTCReadyPromise
  938. = new Promise(r => AdapterJS.webRTCReady(r));
  939. // Resolve or reject depending on whether the Temasys plugin is
  940. // installed.
  941. AdapterJS.WebRTCPlugin.isPluginInstalled(
  942. AdapterJS.WebRTCPlugin.pluginInfo.prefix,
  943. AdapterJS.WebRTCPlugin.pluginInfo.plugName,
  944. AdapterJS.WebRTCPlugin.pluginInfo.type,
  945. /* installed */ () => {
  946. webRTCReadyPromise.then(() => {
  947. webRTCReadyCb();
  948. resolve();
  949. });
  950. },
  951. /* not installed */ () => {
  952. const error
  953. = new Error('Temasys plugin is not installed');
  954. error.name = 'WEBRTC_NOT_READY';
  955. error.webRTCReadyPromise = webRTCReadyPromise;
  956. reject(error);
  957. });
  958. } else {
  959. rejectWithWebRTCNotSupported(
  960. 'Browser does not appear to be WebRTC-capable',
  961. reject);
  962. return;
  963. }
  964. this._initPCConstraints(options);
  965. // Call onReady() if Temasys plugin is not used
  966. if (!browser.isTemasysPluginUsed()) {
  967. onReady(options, this.getUserMediaWithConstraints.bind(this));
  968. resolve();
  969. }
  970. });
  971. }
  972. /**
  973. * Creates instance objects for peer connection constraints both for p2p
  974. * and outside of p2p.
  975. *
  976. * @params {Object} options - Configuration for setting RTCUtil's instance
  977. * objects for peer connection constraints.
  978. * @params {boolean} options.useIPv6 - Set to true if IPv6 should be used.
  979. * @params {boolean} options.disableSuspendVideo - Whether or not video
  980. * should become suspended if bandwidth estimation becomes low.
  981. * @params {Object} options.testing - Additional configuration for work in
  982. * development.
  983. * @params {Object} options.testing.forceP2PSuspendVideoRatio - True if
  984. * video should become suspended if bandwidth estimation becomes low while
  985. * in peer to peer connection mode.
  986. */
  987. _initPCConstraints(options) {
  988. if (browser.isFirefox()) {
  989. this.pcConstraints = {};
  990. } else if (browser.isChrome()
  991. || browser.isOpera()
  992. || browser.isNWJS()
  993. || browser.isElectron()
  994. || browser.isReactNative()) {
  995. this.pcConstraints = { optional: [
  996. { googHighStartBitrate: 0 },
  997. { googPayloadPadding: true },
  998. { googScreencastMinBitrate: 400 },
  999. { googCpuOveruseDetection: true },
  1000. { googCpuOveruseEncodeUsage: true },
  1001. { googCpuUnderuseThreshold: 55 },
  1002. { googCpuOveruseThreshold: 85 }
  1003. ] };
  1004. if (options.useIPv6) {
  1005. // https://code.google.com/p/webrtc/issues/detail?id=2828
  1006. this.pcConstraints.optional.push({ googIPv6: true });
  1007. }
  1008. this.p2pPcConstraints
  1009. = JSON.parse(JSON.stringify(this.pcConstraints));
  1010. // Allows sending of video to be suspended if the bandwidth
  1011. // estimation is too low.
  1012. if (!options.disableSuspendVideo) {
  1013. this.pcConstraints.optional.push(
  1014. { googSuspendBelowMinBitrate: true });
  1015. }
  1016. // There's no reason not to use this for p2p
  1017. this.p2pPcConstraints.optional.push({
  1018. googSuspendBelowMinBitrate: true
  1019. });
  1020. }
  1021. this.p2pPcConstraints = this.p2pPcConstraints || this.pcConstraints;
  1022. }
  1023. /* eslint-disable max-params */
  1024. /**
  1025. * @param {string[]} um required user media types
  1026. * @param {function} successCallback
  1027. * @param {Function} failureCallback
  1028. * @param {Object} [options] optional parameters
  1029. * @param {string} options.resolution
  1030. * @param {number} options.bandwidth
  1031. * @param {number} options.fps
  1032. * @param {string} options.desktopStream
  1033. * @param {string} options.cameraDeviceId
  1034. * @param {string} options.micDeviceId
  1035. * @returns {Promise} Returns a media stream on success or a JitsiTrackError
  1036. * on failure.
  1037. **/
  1038. getUserMediaWithConstraints(
  1039. um,
  1040. successCallback,
  1041. failureCallback,
  1042. options = {}) {
  1043. const constraints = getConstraints(um, options);
  1044. logger.info('Get media constraints', constraints);
  1045. return new Promise((resolve, reject) => {
  1046. try {
  1047. this.getUserMedia(
  1048. constraints,
  1049. stream => {
  1050. logger.log('onUserMediaSuccess');
  1051. setAvailableDevices(um, stream);
  1052. if (successCallback) {
  1053. successCallback(stream);
  1054. }
  1055. resolve(stream);
  1056. },
  1057. error => {
  1058. setAvailableDevices(um, undefined);
  1059. logger.warn(
  1060. 'Failed to get access to local media. Error ',
  1061. error, constraints);
  1062. const jitsiTrackError
  1063. = new JitsiTrackError(error, constraints, um);
  1064. if (failureCallback) {
  1065. failureCallback(jitsiTrackError);
  1066. }
  1067. reject(jitsiTrackError);
  1068. });
  1069. } catch (e) {
  1070. logger.error('GUM failed: ', e);
  1071. const jitsiTrackError
  1072. = new JitsiTrackError(e, constraints, um);
  1073. if (failureCallback) {
  1074. failureCallback(jitsiTrackError);
  1075. }
  1076. reject(jitsiTrackError);
  1077. }
  1078. });
  1079. }
  1080. /**
  1081. * Acquires a media stream via getUserMedia that
  1082. * matches the given constraints
  1083. *
  1084. * @param {array} umDevices which devices to acquire (e.g. audio, video)
  1085. * @param {Object} constraints - Stream specifications to use.
  1086. * @returns {Promise}
  1087. */
  1088. _newGetUserMediaWithConstraints(umDevices, constraints = {}) {
  1089. return new Promise((resolve, reject) => {
  1090. try {
  1091. this.getUserMedia(constraints)
  1092. .then(stream => {
  1093. logger.log('onUserMediaSuccess');
  1094. // TODO(brian): Is this call needed? Why is this
  1095. // happening at gUM time? Isn't there an event listener
  1096. // for this?
  1097. setAvailableDevices(umDevices, stream);
  1098. resolve(stream);
  1099. })
  1100. .catch(error => {
  1101. logger.warn('Failed to get access to local media. '
  1102. + ` ${error} ${constraints} `);
  1103. // TODO(brian): Is this call needed? Why is this
  1104. // happening at gUM time? Isn't there an event listener
  1105. // for this?
  1106. setAvailableDevices(umDevices, undefined);
  1107. reject(new JitsiTrackError(
  1108. error, constraints, umDevices));
  1109. });
  1110. } catch (error) {
  1111. logger.error(`GUM failed: ${error}`);
  1112. reject(new JitsiTrackError(error, constraints, umDevices));
  1113. }
  1114. });
  1115. }
  1116. /**
  1117. * Acquire a display stream via the screenObtainer. This requires extra
  1118. * logic compared to use screenObtainer versus normal device capture logic
  1119. * in RTCUtils#_newGetUserMediaWithConstraints.
  1120. *
  1121. * @param {Object} desktopSharingExtensionExternalInstallation
  1122. * @param {string[]} desktopSharingSources
  1123. * @returns {Promise} A promise which will be resolved with an object whic
  1124. * contains the acquired display stream. If desktop sharing is not supported
  1125. * then a rejected promise will be returned.
  1126. */
  1127. _newGetDesktopMedia(
  1128. desktopSharingExtensionExternalInstallation,
  1129. desktopSharingSources) {
  1130. if (!screenObtainer.isSupported() || !browser.supportsVideo()) {
  1131. return Promise.reject(
  1132. new Error('Desktop sharing is not supported!'));
  1133. }
  1134. const desktopSharingOptions = {
  1135. ...desktopSharingExtensionExternalInstallation,
  1136. desktopSharingSources
  1137. };
  1138. return new Promise((resolve, reject) => {
  1139. screenObtainer.obtainStream(
  1140. desktopSharingOptions,
  1141. stream => {
  1142. resolve(stream);
  1143. },
  1144. error => {
  1145. reject(error);
  1146. });
  1147. });
  1148. }
  1149. /* eslint-enable max-params */
  1150. /**
  1151. * Creates the local MediaStreams.
  1152. * @param {Object} [options] optional parameters
  1153. * @param {Array} options.devices the devices that will be requested
  1154. * @param {string} options.resolution resolution constraints
  1155. * @param {bool} options.dontCreateJitsiTrack if <tt>true</tt> objects with
  1156. * the following structure {stream: the Media Stream, type: "audio" or
  1157. * "video", videoType: "camera" or "desktop"} will be returned trough the
  1158. * Promise, otherwise JitsiTrack objects will be returned.
  1159. * @param {string} options.cameraDeviceId
  1160. * @param {string} options.micDeviceId
  1161. * @returns {*} Promise object that will receive the new JitsiTracks
  1162. */
  1163. obtainAudioAndVideoPermissions(options = {}) {
  1164. options.devices = options.devices || [ ...OLD_GUM_DEFAULT_DEVICES ];
  1165. options.resolution = options.resolution || OLD_GUM_DEFAULT_RESOLUTION;
  1166. const requestingDesktop = options.devices.includes('desktop');
  1167. if (requestingDesktop && !screenObtainer.isSupported()) {
  1168. return Promise.reject(
  1169. new Error('Desktop sharing is not supported!'));
  1170. }
  1171. let gumPromise;
  1172. if (browser.supportsMediaStreamConstructor()) {
  1173. gumPromise = this._getAudioAndVideoStreams(options);
  1174. } else {
  1175. // If the MediaStream constructor is not supported, then get tracks
  1176. // in separate GUM calls in order to keep different tracks separate.
  1177. gumPromise = this._getAudioAndVideoStreamsSeparately(options);
  1178. }
  1179. return gumPromise.then(streams =>
  1180. handleLocalStream(streams, options.resolution));
  1181. }
  1182. /**
  1183. * Performs one call to getUserMedia for audio and/or video and another call
  1184. * for desktop.
  1185. *
  1186. * @param {Object} options - An object describing how the gUM request should
  1187. * be executed. See {@link obtainAudioAndVideoPermissions} for full options.
  1188. * @returns {*} Promise object that will receive the new JitsiTracks on
  1189. * success or a JitsiTrackError on failure.
  1190. */
  1191. _getAudioAndVideoStreams(options) {
  1192. const requestingDesktop = options.devices.includes('desktop');
  1193. options.devices = options.devices.filter(device =>
  1194. device !== 'desktop');
  1195. const gumPromise = options.devices.length
  1196. ? this.getUserMediaWithConstraints(
  1197. options.devices, null, null, options)
  1198. : Promise.resolve(null);
  1199. return gumPromise
  1200. .then(avStream => {
  1201. // If any requested devices are missing, call gum again in
  1202. // an attempt to obtain the actual error. For example, the
  1203. // requested video device is missing or permission was
  1204. // denied.
  1205. const missingTracks
  1206. = this._getMissingTracks(options.devices, avStream);
  1207. if (missingTracks.length) {
  1208. this.stopMediaStream(avStream);
  1209. return this.getUserMediaWithConstraints(
  1210. missingTracks, options)
  1211. // GUM has already failed earlier and this success
  1212. // handling should not be reached.
  1213. .then(() => Promise.reject(new JitsiTrackError(
  1214. { name: 'UnknownError' },
  1215. getConstraints(options.devices, options),
  1216. missingTracks)));
  1217. }
  1218. return avStream;
  1219. })
  1220. .then(audioVideo => {
  1221. if (!requestingDesktop) {
  1222. return { audioVideo };
  1223. }
  1224. return new Promise((resolve, reject) => {
  1225. screenObtainer.obtainStream(
  1226. this._parseDesktopSharingOptions(options),
  1227. desktop => resolve({
  1228. audioVideo,
  1229. desktop
  1230. }),
  1231. error => {
  1232. if (audioVideo) {
  1233. this.stopMediaStream(audioVideo);
  1234. }
  1235. reject(error);
  1236. });
  1237. });
  1238. });
  1239. }
  1240. /**
  1241. * Private utility for determining if the passed in MediaStream contains
  1242. * tracks of the type(s) specified in the requested devices.
  1243. *
  1244. * @param {string[]} requestedDevices - The track types that are expected to
  1245. * be includes in the stream.
  1246. * @param {MediaStream} stream - The MediaStream to check if it has the
  1247. * expected track types.
  1248. * @returns {string[]} An array of string with the missing track types. The
  1249. * array will be empty if all requestedDevices are found in the stream.
  1250. */
  1251. _getMissingTracks(requestedDevices = [], stream) {
  1252. const missingDevices = [];
  1253. const audioDeviceRequested = requestedDevices.includes('audio');
  1254. const audioTracksReceived
  1255. = stream && stream.getAudioTracks().length > 0;
  1256. if (audioDeviceRequested && !audioTracksReceived) {
  1257. missingDevices.push('audio');
  1258. }
  1259. const videoDeviceRequested = requestedDevices.includes('video');
  1260. const videoTracksReceived
  1261. = stream && stream.getVideoTracks().length > 0;
  1262. if (videoDeviceRequested && !videoTracksReceived) {
  1263. missingDevices.push('video');
  1264. }
  1265. return missingDevices;
  1266. }
  1267. /**
  1268. * Performs separate getUserMedia calls for audio and video instead of in
  1269. * one call. Will also request desktop if specified.
  1270. *
  1271. * @param {Object} options - An object describing how the gUM request should
  1272. * be executed. See {@link obtainAudioAndVideoPermissions} for full options.
  1273. * @returns {*} Promise object that will receive the new JitsiTracks on
  1274. * success or a JitsiTrackError on failure.
  1275. */
  1276. _getAudioAndVideoStreamsSeparately(options) {
  1277. return new Promise((resolve, reject) => {
  1278. const deviceGUM = {
  1279. audio: (...args) =>
  1280. this.getUserMediaWithConstraints([ 'audio' ], ...args),
  1281. video: (...args) =>
  1282. this.getUserMediaWithConstraints([ 'video' ], ...args),
  1283. desktop: (...args) =>
  1284. screenObtainer.obtainStream(
  1285. this._parseDesktopSharingOptions(options), ...args)
  1286. };
  1287. obtainDevices({
  1288. devices: options.devices,
  1289. streams: [],
  1290. successCallback: resolve,
  1291. errorCallback: reject,
  1292. deviceGUM
  1293. });
  1294. });
  1295. }
  1296. /**
  1297. * Returns an object formatted for specifying desktop sharing parameters.
  1298. *
  1299. * @param {Object} options - Takes in the same options object as
  1300. * {@link obtainAudioAndVideoPermissions}.
  1301. * @returns {Object}
  1302. */
  1303. _parseDesktopSharingOptions(options) {
  1304. return {
  1305. ...options.desktopSharingExtensionExternalInstallation,
  1306. desktopSharingSources: options.desktopSharingSources
  1307. };
  1308. }
  1309. /**
  1310. * Gets streams from specified device types. This function intentionally
  1311. * ignores errors for upstream to catch and handle instead.
  1312. *
  1313. * @param {Object} options - A hash describing what devices to get and
  1314. * relevant constraints.
  1315. * @param {string[]} options.devices - The types of media to capture. Valid
  1316. * values are "desktop", "audio", and "video".
  1317. * @returns {Promise} The promise, when successful, will return an array of
  1318. * meta data for the requested device type, which includes the stream and
  1319. * track. If an error occurs, it will be deferred to the caller for
  1320. * handling.
  1321. */
  1322. newObtainAudioAndVideoPermissions(options) {
  1323. logger.info('Using the new gUM flow');
  1324. const mediaStreamsMetaData = [];
  1325. // Declare private functions to be used in the promise chain below.
  1326. // These functions are declared in the scope of this function because
  1327. // they are not being used anywhere else, so only this function needs to
  1328. // know about them.
  1329. /**
  1330. * Executes a request for desktop media if specified in options.
  1331. *
  1332. * @returns {Promise}
  1333. */
  1334. const maybeRequestDesktopDevice = function() {
  1335. const umDevices = options.devices || [];
  1336. const isDesktopDeviceRequsted = umDevices.indexOf('desktop') !== -1;
  1337. return isDesktopDeviceRequsted
  1338. ? this._newGetDesktopMedia(
  1339. options.desktopSharingExtensionExternalInstallation,
  1340. options.desktopSharingSources)
  1341. : Promise.resolve();
  1342. }.bind(this);
  1343. /**
  1344. * Creates a meta data object about the passed in desktopStream and
  1345. * pushes the meta data to the internal array mediaStreamsMetaData to be
  1346. * returned later.
  1347. *
  1348. * @param {MediaStreamTrack} desktopStream - A track for a desktop
  1349. * capture.
  1350. * @returns {void}
  1351. */
  1352. const maybeCreateAndAddDesktopTrack = function(desktopStream) {
  1353. if (!desktopStream) {
  1354. return;
  1355. }
  1356. const { stream, sourceId, sourceType } = desktopStream;
  1357. mediaStreamsMetaData.push({
  1358. stream,
  1359. sourceId,
  1360. sourceType,
  1361. track: stream.getVideoTracks()[0],
  1362. videoType: VideoType.DESKTOP
  1363. });
  1364. };
  1365. /**
  1366. * Executes a request for audio and/or video, as specified in options.
  1367. * By default both audio and video will be captured if options.devices
  1368. * is not defined.
  1369. *
  1370. * @returns {Promise}
  1371. */
  1372. const maybeRequestCaptureDevices = function() {
  1373. const umDevices = options.devices || [ 'audio', 'video' ];
  1374. const requestedCaptureDevices = umDevices.filter(device =>
  1375. device === 'audio'
  1376. || (device === 'video' && browser.supportsVideo()));
  1377. if (!requestedCaptureDevices.length) {
  1378. return Promise.resolve();
  1379. }
  1380. const constraints = newGetConstraints(
  1381. requestedCaptureDevices, options);
  1382. logger.info('Got media constraints: ', constraints);
  1383. return this._newGetUserMediaWithConstraints(
  1384. requestedCaptureDevices, constraints);
  1385. }.bind(this);
  1386. /**
  1387. * Splits the passed in media stream into separate audio and video
  1388. * streams and creates meta data objects for each and pushes them to the
  1389. * internal array mediaStreamsMetaData to be returned later.
  1390. *
  1391. * @param {MediaStreamTrack} avStream - A track for with audio and/or
  1392. * video track.
  1393. * @returns {void}
  1394. */
  1395. const maybeCreateAndAddAVTracks = function(avStream) {
  1396. if (!avStream) {
  1397. return;
  1398. }
  1399. const audioTracks = avStream.getAudioTracks();
  1400. if (audioTracks.length) {
  1401. const audioStream = new MediaStream(audioTracks);
  1402. mediaStreamsMetaData.push({
  1403. stream: audioStream,
  1404. track: audioStream.getAudioTracks()[0]
  1405. });
  1406. }
  1407. const videoTracks = avStream.getVideoTracks();
  1408. if (videoTracks.length) {
  1409. const videoStream = new MediaStream(videoTracks);
  1410. mediaStreamsMetaData.push({
  1411. stream: videoStream,
  1412. track: videoStream.getVideoTracks()[0],
  1413. videoType: VideoType.CAMERA
  1414. });
  1415. }
  1416. };
  1417. return maybeRequestDesktopDevice()
  1418. .then(maybeCreateAndAddDesktopTrack)
  1419. .then(maybeRequestCaptureDevices)
  1420. .then(maybeCreateAndAddAVTracks)
  1421. .then(() => mediaStreamsMetaData);
  1422. }
  1423. /**
  1424. *
  1425. */
  1426. getDeviceAvailability() {
  1427. return devices;
  1428. }
  1429. /**
  1430. *
  1431. */
  1432. isRTCReady() {
  1433. return rtcReady;
  1434. }
  1435. /**
  1436. *
  1437. */
  1438. _isDeviceListAvailable() {
  1439. if (!rtcReady) {
  1440. throw new Error('WebRTC not ready yet');
  1441. }
  1442. return Boolean(
  1443. (navigator.mediaDevices
  1444. && navigator.mediaDevices.enumerateDevices)
  1445. || (typeof MediaStreamTrack !== 'undefined'
  1446. && MediaStreamTrack.getSources));
  1447. }
  1448. /**
  1449. * Returns a promise which can be used to make sure that the WebRTC stack
  1450. * has been initialized.
  1451. *
  1452. * @returns {Promise} which is resolved only if the WebRTC stack is ready.
  1453. * Note that currently we do not detect stack initialization failure and
  1454. * the promise is never rejected(unless unexpected error occurs).
  1455. */
  1456. onRTCReady() {
  1457. if (rtcReady) {
  1458. return Promise.resolve();
  1459. }
  1460. return new Promise(resolve => {
  1461. const listener = () => {
  1462. eventEmitter.removeListener(RTCEvents.RTC_READY, listener);
  1463. resolve();
  1464. };
  1465. eventEmitter.addListener(RTCEvents.RTC_READY, listener);
  1466. // We have no failed event, so... it either resolves or nothing
  1467. // happens.
  1468. });
  1469. }
  1470. /**
  1471. * Checks if its possible to enumerate available cameras/microphones.
  1472. *
  1473. * @returns {Promise<boolean>} a Promise which will be resolved only once
  1474. * the WebRTC stack is ready, either with true if the device listing is
  1475. * available available or with false otherwise.
  1476. */
  1477. isDeviceListAvailable() {
  1478. return this.onRTCReady().then(this._isDeviceListAvailable.bind(this));
  1479. }
  1480. /**
  1481. * Returns true if changing the input (camera / microphone) or output
  1482. * (audio) device is supported and false if not.
  1483. * @params {string} [deviceType] - type of device to change. Default is
  1484. * undefined or 'input', 'output' - for audio output device change.
  1485. * @returns {boolean} true if available, false otherwise.
  1486. */
  1487. isDeviceChangeAvailable(deviceType) {
  1488. return deviceType === 'output' || deviceType === 'audiooutput'
  1489. ? isAudioOutputDeviceChangeAvailable
  1490. : browser.isChrome()
  1491. || browser.isFirefox()
  1492. || browser.isOpera()
  1493. || browser.isTemasysPluginUsed()
  1494. || browser.isNWJS()
  1495. || browser.isElectron()
  1496. || browser.isEdge();
  1497. }
  1498. /**
  1499. * A method to handle stopping of the stream.
  1500. * One point to handle the differences in various implementations.
  1501. * @param mediaStream MediaStream object to stop.
  1502. */
  1503. stopMediaStream(mediaStream) {
  1504. mediaStream.getTracks().forEach(track => {
  1505. // stop() not supported with IE
  1506. if (!browser.isTemasysPluginUsed() && track.stop) {
  1507. track.stop();
  1508. }
  1509. });
  1510. // leave stop for implementation still using it
  1511. if (mediaStream.stop) {
  1512. mediaStream.stop();
  1513. }
  1514. // The MediaStream implementation of the react-native-webrtc project has
  1515. // an explicit release method that is to be invoked in order to release
  1516. // used resources such as memory.
  1517. if (mediaStream.release) {
  1518. mediaStream.release();
  1519. }
  1520. // if we have done createObjectURL, lets clean it
  1521. const url = mediaStream.jitsiObjectURL;
  1522. if (url) {
  1523. delete mediaStream.jitsiObjectURL;
  1524. (URL || webkitURL).revokeObjectURL(url);
  1525. }
  1526. }
  1527. /**
  1528. * Returns whether the desktop sharing is enabled or not.
  1529. * @returns {boolean}
  1530. */
  1531. isDesktopSharingEnabled() {
  1532. return screenObtainer.isSupported();
  1533. }
  1534. /**
  1535. * Sets current audio output device.
  1536. * @param {string} deviceId - id of 'audiooutput' device from
  1537. * navigator.mediaDevices.enumerateDevices(), 'default' for default
  1538. * device
  1539. * @returns {Promise} - resolves when audio output is changed, is rejected
  1540. * otherwise
  1541. */
  1542. setAudioOutputDevice(deviceId) {
  1543. if (!this.isDeviceChangeAvailable('output')) {
  1544. Promise.reject(
  1545. new Error('Audio output device change is not supported'));
  1546. }
  1547. return featureDetectionAudioEl.setSinkId(deviceId)
  1548. .then(() => {
  1549. audioOutputDeviceId = deviceId;
  1550. audioOutputChanged = true;
  1551. logger.log(`Audio output device set to ${deviceId}`);
  1552. eventEmitter.emit(RTCEvents.AUDIO_OUTPUT_DEVICE_CHANGED,
  1553. deviceId);
  1554. });
  1555. }
  1556. /**
  1557. * Returns currently used audio output device id, '' stands for default
  1558. * device
  1559. * @returns {string}
  1560. */
  1561. getAudioOutputDevice() {
  1562. return audioOutputDeviceId;
  1563. }
  1564. /**
  1565. * Returns list of available media devices if its obtained, otherwise an
  1566. * empty array is returned/
  1567. * @returns {Array} list of available media devices.
  1568. */
  1569. getCurrentlyAvailableMediaDevices() {
  1570. return currentlyAvailableMediaDevices;
  1571. }
  1572. /**
  1573. * Returns event data for device to be reported to stats.
  1574. * @returns {MediaDeviceInfo} device.
  1575. */
  1576. getEventDataForActiveDevice(device) {
  1577. const deviceList = [];
  1578. const deviceData = {
  1579. 'deviceId': device.deviceId,
  1580. 'kind': device.kind,
  1581. 'label': device.label,
  1582. 'groupId': device.groupId
  1583. };
  1584. deviceList.push(deviceData);
  1585. return { deviceList };
  1586. }
  1587. /**
  1588. * Configures the given PeerConnection constraints to either enable or
  1589. * disable (according to the value of the 'enable' parameter) the
  1590. * 'googSuspendBelowMinBitrate' option.
  1591. * @param constraints the constraints on which to operate.
  1592. * @param enable {boolean} whether to enable or disable the suspend video
  1593. * option.
  1594. */
  1595. setSuspendVideo(constraints, enable) {
  1596. if (!constraints.optional) {
  1597. constraints.optional = [];
  1598. }
  1599. // Get rid of all "googSuspendBelowMinBitrate" constraints (we assume
  1600. // that the elements of constraints.optional contain a single property).
  1601. constraints.optional
  1602. = constraints.optional.filter(
  1603. c => !c.hasOwnProperty('googSuspendBelowMinBitrate'));
  1604. if (enable) {
  1605. constraints.optional.push({ googSuspendBelowMinBitrate: 'true' });
  1606. }
  1607. }
  1608. }
  1609. /**
  1610. * Rejects a Promise because WebRTC is not supported.
  1611. *
  1612. * @param {string} errorMessage - The human-readable message of the Error which
  1613. * is the reason for the rejection.
  1614. * @param {Function} reject - The reject function of the Promise.
  1615. * @returns {void}
  1616. */
  1617. function rejectWithWebRTCNotSupported(errorMessage, reject) {
  1618. const error = new Error(errorMessage);
  1619. // WebRTC is not supported either natively or via a known plugin such as
  1620. // Temasys.
  1621. // XXX The Error class already has a property name which is commonly used to
  1622. // detail the represented error in a non-human-readable way (in contrast to
  1623. // the human-readable property message). I explicitly did not want to
  1624. // introduce a new specific property.
  1625. // FIXME None of the existing JitsiXXXErrors seemed to be appropriate
  1626. // recipients of the constant WEBRTC_NOT_SUPPORTED so I explicitly chose to
  1627. // leave it as a magic string at the time of this writing.
  1628. error.name = 'WEBRTC_NOT_SUPPORTED';
  1629. logger.error(errorMessage);
  1630. reject(error);
  1631. }
  1632. const rtcUtils = new RTCUtils();
  1633. /**
  1634. *
  1635. * @param options
  1636. */
  1637. function obtainDevices(options) {
  1638. if (!options.devices || options.devices.length === 0) {
  1639. return options.successCallback(options.streams || {});
  1640. }
  1641. const device = options.devices.splice(0, 1);
  1642. options.deviceGUM[device](
  1643. stream => {
  1644. options.streams = options.streams || {};
  1645. options.streams[device] = stream;
  1646. obtainDevices(options);
  1647. },
  1648. error => {
  1649. Object.keys(options.streams).forEach(
  1650. d => rtcUtils.stopMediaStream(options.streams[d]));
  1651. logger.error(
  1652. `failed to obtain ${device} stream - stop`, error);
  1653. options.errorCallback(error);
  1654. });
  1655. }
  1656. /**
  1657. * In case of IE we continue from 'onReady' callback passed to RTCUtils
  1658. * constructor. It will be invoked by Temasys plugin once it is initialized.
  1659. *
  1660. * @param options
  1661. * @param GUM
  1662. */
  1663. function onReady(options, GUM) {
  1664. rtcReady = true;
  1665. eventEmitter.emit(RTCEvents.RTC_READY, true);
  1666. screenObtainer.init(options, GUM);
  1667. if (rtcUtils.isDeviceListAvailable() && rawEnumerateDevicesWithCallback) {
  1668. rawEnumerateDevicesWithCallback(ds => {
  1669. currentlyAvailableMediaDevices = ds.splice(0);
  1670. logger.info('Available devices: ', currentlyAvailableMediaDevices);
  1671. sendDeviceListToAnalytics(currentlyAvailableMediaDevices);
  1672. eventEmitter.emit(RTCEvents.DEVICE_LIST_AVAILABLE,
  1673. currentlyAvailableMediaDevices);
  1674. if (isDeviceChangeEventSupported) {
  1675. navigator.mediaDevices.addEventListener(
  1676. 'devicechange',
  1677. () => rtcUtils.enumerateDevices(onMediaDevicesListChanged));
  1678. } else {
  1679. pollForAvailableMediaDevices();
  1680. }
  1681. });
  1682. }
  1683. }
  1684. /**
  1685. * Wraps original attachMediaStream function to set current audio output device
  1686. * if this is supported.
  1687. * @param {Function} origAttachMediaStream
  1688. * @returns {Function}
  1689. */
  1690. function wrapAttachMediaStream(origAttachMediaStream) {
  1691. return function(element, stream) {
  1692. // eslint-disable-next-line prefer-rest-params
  1693. const res = origAttachMediaStream.apply(rtcUtils, arguments);
  1694. if (stream
  1695. && rtcUtils.isDeviceChangeAvailable('output')
  1696. && stream.getAudioTracks
  1697. && stream.getAudioTracks().length
  1698. // we skip setting audio output if there was no explicit change
  1699. && audioOutputChanged) {
  1700. element.setSinkId(rtcUtils.getAudioOutputDevice())
  1701. .catch(function(ex) {
  1702. const err
  1703. = new JitsiTrackError(ex, null, [ 'audiooutput' ]);
  1704. GlobalOnErrorHandler.callUnhandledRejectionHandler({
  1705. promise: this, // eslint-disable-line no-invalid-this
  1706. reason: err
  1707. });
  1708. logger.warn(
  1709. 'Failed to set audio output device for the element.'
  1710. + ' Default audio output device will be used'
  1711. + ' instead',
  1712. element,
  1713. err);
  1714. });
  1715. }
  1716. return res;
  1717. };
  1718. }
  1719. export default rtcUtils;