Nevar pievienot vairāk kā 25 tēmas Tēmai ir jāsākas ar burtu vai ciparu, tā var saturēt domu zīmes ('-') un var būt līdz 35 simboliem gara.

TPCUtils.js 35KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813
  1. import { getLogger } from '@jitsi/logger';
  2. import clonedeep from 'lodash.clonedeep';
  3. import transform from 'sdp-transform';
  4. import CodecMimeType from '../../service/RTC/CodecMimeType';
  5. import { MediaDirection } from '../../service/RTC/MediaDirection';
  6. import { MediaType } from '../../service/RTC/MediaType';
  7. import { getSourceIndexFromSourceName } from '../../service/RTC/SignalingLayer';
  8. import { STANDARD_CODEC_SETTINGS } from '../../service/RTC/StandardVideoSettings';
  9. import VideoEncoderScalabilityMode from '../../service/RTC/VideoEncoderScalabilityMode';
  10. import { VideoType } from '../../service/RTC/VideoType';
  11. import browser from '../browser';
  12. const logger = getLogger(__filename);
  13. const DESKTOP_SHARE_RATE = 500000;
  14. const SIM_LAYER_1_RID = '1';
  15. const SIM_LAYER_2_RID = '2';
  16. const SIM_LAYER_3_RID = '3';
  17. const VIDEO_CODECS = [ CodecMimeType.AV1, CodecMimeType.H264, CodecMimeType.VP8, CodecMimeType.VP9 ];
  18. // TODO - need to revisit these settings when 4K is the captured resolution. We can change the LD scale factor to 6
  19. // instead of 4 so that the lowest resolution will be 360p instead of 540p.
  20. export const HD_SCALE_FACTOR = 1.0;
  21. export const LD_SCALE_FACTOR = 4.0;
  22. export const SD_SCALE_FACTOR = 2.0;
  23. export const SIM_LAYER_RIDS = [ SIM_LAYER_1_RID, SIM_LAYER_2_RID, SIM_LAYER_3_RID ];
  24. /**
  25. * Handles track related operations on TraceablePeerConnection when browser is
  26. * running in unified plan mode.
  27. */
  28. export class TPCUtils {
  29. /**
  30. * Creates a new instance for a given TraceablePeerConnection
  31. *
  32. * @param peerconnection - the tpc instance for which we have utility functions.
  33. */
  34. constructor(peerconnection) {
  35. this.pc = peerconnection;
  36. this.codecSettings = clonedeep(STANDARD_CODEC_SETTINGS);
  37. const videoQualitySettings = this.pc.options?.videoQuality;
  38. if (videoQualitySettings) {
  39. for (const codec of VIDEO_CODECS) {
  40. const codecConfig = videoQualitySettings[codec];
  41. const bitrateSettings = codecConfig?.maxBitratesVideo
  42. // Read the deprecated settings for max bitrates.
  43. ?? (videoQualitySettings.maxbitratesvideo
  44. && videoQualitySettings.maxbitratesvideo[codec.toUpperCase()]);
  45. if (bitrateSettings) {
  46. [ 'low', 'standard', 'high', 'ssHigh' ].forEach(value => {
  47. if (bitrateSettings[value]) {
  48. this.codecSettings[codec].maxBitratesVideo[value] = bitrateSettings[value];
  49. }
  50. });
  51. }
  52. if (!codecConfig) {
  53. continue; // eslint-disable-line no-continue
  54. }
  55. const scalabilityModeEnabled = this.codecSettings[codec].scalabilityModeEnabled
  56. && (typeof codecConfig.scalabilityModeEnabled === 'undefined'
  57. || codecConfig.scalabilityModeEnabled);
  58. if (scalabilityModeEnabled) {
  59. typeof codecConfig.useSimulcast !== 'undefined'
  60. && (this.codecSettings[codec].useSimulcast = codecConfig.useSimulcast);
  61. typeof codecConfig.useKSVC !== 'undefined'
  62. && (this.codecSettings[codec].useKSVC = codecConfig.useKSVC);
  63. } else {
  64. this.codecSettings[codec].scalabilityModeEnabled = false;
  65. }
  66. }
  67. }
  68. }
  69. /**
  70. * Calculates the configuration of the active encoding when the browser sends only one stream, i,e,, when there is
  71. * no spatial scalability configure (p2p) or when it is running in full SVC mode.
  72. *
  73. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  74. * @param {CodecMimeType} codec - The video codec.
  75. * @param {number} newHeight - The resolution that needs to be configured for the local video track.
  76. * @returns {Object} configuration.
  77. */
  78. _calculateActiveEncodingParams(localVideoTrack, codec, newHeight) {
  79. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  80. const height = localVideoTrack.getHeight();
  81. const desktopShareBitrate = this.pc.options?.videoQuality?.desktopbitrate || DESKTOP_SHARE_RATE;
  82. const isScreenshare = localVideoTrack.getVideoType() === VideoType.DESKTOP;
  83. let scalabilityMode = this.codecSettings[codec].useKSVC
  84. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3;
  85. let maxBitrate = codecBitrates.high;
  86. if (this._isScreenshareBitrateCapped(localVideoTrack)) {
  87. scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  88. maxBitrate = desktopShareBitrate;
  89. } else if (localVideoTrack.getVideoType() === VideoType.DESKTOP) {
  90. maxBitrate = codecBitrates.ssHigh;
  91. }
  92. const config = {
  93. active: newHeight > 0,
  94. maxBitrate,
  95. scalabilityMode,
  96. scaleResolutionDownBy: HD_SCALE_FACTOR
  97. };
  98. if (newHeight >= height || newHeight === 0 || isScreenshare) {
  99. return config;
  100. }
  101. if (newHeight >= height / SD_SCALE_FACTOR) {
  102. config.maxBitrate = codecBitrates.standard;
  103. config.scalabilityMode = this.codecSettings[codec].useKSVC
  104. ? VideoEncoderScalabilityMode.L2T3_KEY : VideoEncoderScalabilityMode.L2T3;
  105. config.scaleResolutionDownBy = SD_SCALE_FACTOR;
  106. } else {
  107. config.maxBitrate = codecBitrates.low;
  108. config.scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  109. config.scaleResolutionDownBy = LD_SCALE_FACTOR;
  110. }
  111. return config;
  112. }
  113. /**
  114. * Obtains stream encodings that need to be configured on the given track based
  115. * on the track media type and the simulcast setting.
  116. * @param {JitsiLocalTrack} localTrack
  117. */
  118. _getStreamEncodings(localTrack) {
  119. const codec = this.pc.getConfiguredVideoCodec();
  120. const encodings = this._getVideoStreamEncodings(localTrack.getVideoType(), codec);
  121. if (this.pc.isSpatialScalabilityOn() && localTrack.isVideoTrack()) {
  122. return encodings;
  123. }
  124. return localTrack.isVideoTrack()
  125. ? [ {
  126. active: this.pc.videoTransferActive,
  127. maxBitrate: this.codecSettings[codec].maxBitratesVideo.high
  128. } ]
  129. : [ { active: this.pc.audioTransferActive } ];
  130. }
  131. /**
  132. * The startup configuration for the stream encodings that are applicable to
  133. * the video stream when a new sender is created on the peerconnection. The initial
  134. * config takes into account the differences in browser's simulcast implementation.
  135. *
  136. * Encoding parameters:
  137. * active - determine the on/off state of a particular encoding.
  138. * maxBitrate - max. bitrate value to be applied to that particular encoding
  139. * based on the encoding's resolution and config.js videoQuality settings if applicable.
  140. * rid - Rtp Stream ID that is configured for a particular simulcast stream.
  141. * scaleResolutionDownBy - the factor by which the encoding is scaled down from the
  142. * original resolution of the captured video.
  143. *
  144. * @param {VideoType} videoType
  145. * @param {String} codec
  146. */
  147. _getVideoStreamEncodings(videoType, codec) {
  148. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  149. const maxVideoBitrate = videoType === VideoType.DESKTOP
  150. ? codecBitrates.ssHigh : codecBitrates.high;
  151. // The SSRCs on older versions of Firefox are reversed in SDP, i.e., they have resolution order of 1:2:4 as
  152. // opposed to Chromium and other browsers. This has been reverted in Firefox 117 as part of the below commit.
  153. // https://hg.mozilla.org/mozilla-central/rev/b0348f1f8d7197fb87158ba74542d28d46133997
  154. // This revert seems to be applied only to camera tracks, the desktop stream encodings still have the
  155. // resolution order of 4:2:1.
  156. const reversedEncodings = browser.isFirefox()
  157. && (videoType === VideoType.DESKTOP || browser.isVersionLessThan(117));
  158. const standardSimulcastEncodings = [
  159. {
  160. active: this.pc.videoTransferActive,
  161. maxBitrate: reversedEncodings ? maxVideoBitrate : codecBitrates.low,
  162. rid: SIM_LAYER_1_RID,
  163. scaleResolutionDownBy: reversedEncodings ? HD_SCALE_FACTOR : LD_SCALE_FACTOR
  164. },
  165. {
  166. active: this.pc.videoTransferActive,
  167. maxBitrate: codecBitrates.standard,
  168. rid: SIM_LAYER_2_RID,
  169. scaleResolutionDownBy: SD_SCALE_FACTOR
  170. },
  171. {
  172. active: this.pc.videoTransferActive,
  173. maxBitrate: reversedEncodings ? codecBitrates.low : maxVideoBitrate,
  174. rid: SIM_LAYER_3_RID,
  175. scaleResolutionDownBy: reversedEncodings ? LD_SCALE_FACTOR : HD_SCALE_FACTOR
  176. }
  177. ];
  178. if (this.codecSettings[codec].scalabilityModeEnabled) {
  179. // Configure all 3 encodings when simulcast is requested through config.js for AV1 and VP9 and for H.264
  180. // always since that is the only supported mode when DD header extension is negotiated for H.264.
  181. if (this.codecSettings[codec].useSimulcast || codec === CodecMimeType.H264) {
  182. for (const encoding of standardSimulcastEncodings) {
  183. encoding.scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  184. }
  185. return standardSimulcastEncodings;
  186. }
  187. // Configure only one encoding for the SVC mode.
  188. return [
  189. {
  190. active: this.pc.videoTransferActive,
  191. maxBitrate: maxVideoBitrate,
  192. rid: SIM_LAYER_1_RID,
  193. scaleResolutionDownBy: HD_SCALE_FACTOR,
  194. scalabilityMode: this.codecSettings[codec].useKSVC
  195. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3
  196. },
  197. {
  198. active: false,
  199. maxBitrate: 0
  200. },
  201. {
  202. active: false,
  203. maxBitrate: 0
  204. }
  205. ];
  206. }
  207. return standardSimulcastEncodings;
  208. }
  209. /**
  210. * Returns a boolean indicating whether the video encoder is running in full SVC mode, i.e., it sends only one
  211. * video stream that has both temporal and spatial scalability.
  212. *
  213. * @param {CodecMimeType} codec
  214. * @returns boolean
  215. */
  216. _isRunningInFullSvcMode(codec) {
  217. return (codec === CodecMimeType.VP9 || codec === CodecMimeType.AV1)
  218. && this.codecSettings[codec].scalabilityModeEnabled
  219. && !this.codecSettings[codec].useSimulcast;
  220. }
  221. /**
  222. * Returns a boolean indicating whether the bitrate needs to be capped for the local video track if it happens to
  223. * be a screenshare track. The lower spatial layers for screensharing are disabled when low fps screensharing is in
  224. * progress. Sending all three streams often results in the browser suspending the high resolution in low b/w and
  225. * and low cpu conditions, especially on the low end machines. Suspending the low resolution streams ensures that
  226. * the highest resolution stream is available always. Safari is an exception here since it does not send the
  227. * desktop stream at all if only the high resolution stream is enabled.
  228. *
  229. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  230. * @returns {boolean}
  231. */
  232. _isScreenshareBitrateCapped(localVideoTrack) {
  233. return localVideoTrack.getVideoType() === VideoType.DESKTOP
  234. && this.pc._capScreenshareBitrate
  235. && !browser.isWebKitBased();
  236. }
  237. /**
  238. * Updates the sender parameters in the stream encodings.
  239. *
  240. * @param {RTCRtpSender} sender - the sender associated with a MediaStreamTrack.
  241. * @param {boolean} enable - whether the streams needs to be enabled or disabled.
  242. * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise.
  243. */
  244. _updateSenderEncodings(sender, enable) {
  245. const parameters = sender.getParameters();
  246. if (parameters?.encodings?.length) {
  247. for (const encoding of parameters.encodings) {
  248. encoding.active = enable;
  249. }
  250. }
  251. return sender.setParameters(parameters);
  252. }
  253. /**
  254. * Adds {@link JitsiLocalTrack} to the WebRTC peerconnection for the first time.
  255. * @param {JitsiLocalTrack} track - track to be added to the peerconnection.
  256. * @param {boolean} isInitiator - boolean that indicates if the endpoint is offerer in a p2p connection.
  257. * @returns {void}
  258. */
  259. addTrack(localTrack, isInitiator) {
  260. const track = localTrack.getTrack();
  261. if (isInitiator) {
  262. const streams = [];
  263. if (localTrack.getOriginalStream()) {
  264. streams.push(localTrack.getOriginalStream());
  265. }
  266. // Use pc.addTransceiver() for the initiator case when local tracks are getting added
  267. // to the peerconnection before a session-initiate is sent over to the peer.
  268. const transceiverInit = {
  269. direction: MediaDirection.SENDRECV,
  270. streams,
  271. sendEncodings: []
  272. };
  273. if (!browser.isFirefox()) {
  274. transceiverInit.sendEncodings = this._getStreamEncodings(localTrack);
  275. }
  276. this.pc.peerconnection.addTransceiver(track, transceiverInit);
  277. } else {
  278. // Use pc.addTrack() for responder case so that we can re-use the m-lines that were created
  279. // when setRemoteDescription was called. pc.addTrack() automatically attaches to any existing
  280. // unused "recv-only" transceiver.
  281. this.pc.peerconnection.addTrack(track);
  282. }
  283. }
  284. /**
  285. * Returns the calculated active state of the stream encodings based on the frame height requested for the send
  286. * stream. All the encodings that have a resolution lower than the frame height requested will be enabled.
  287. *
  288. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  289. * @param {CodecMimeType} codec - The codec currently in use.
  290. * @param {number} newHeight The resolution requested for the video track.
  291. * @returns {Array<boolean>}
  292. */
  293. calculateEncodingsActiveState(localVideoTrack, codec, newHeight) {
  294. const height = localVideoTrack.getHeight();
  295. const videoStreamEncodings = this._getVideoStreamEncodings(localVideoTrack.getVideoType(), codec);
  296. const encodingsState = videoStreamEncodings
  297. .map(encoding => height / encoding.scaleResolutionDownBy)
  298. .map((frameHeight, idx) => {
  299. // Single video stream.
  300. if (!this.pc.isSpatialScalabilityOn() || this._isRunningInFullSvcMode(codec)) {
  301. const { active } = this._calculateActiveEncodingParams(localVideoTrack, codec, newHeight);
  302. return idx === 0 ? active : false;
  303. }
  304. // Multiple video streams.
  305. let active = false;
  306. if (newHeight > 0) {
  307. if (localVideoTrack.getVideoType() === VideoType.CAMERA) {
  308. active = frameHeight <= newHeight
  309. // Keep the LD stream enabled even when the LD stream's resolution is higher than of the
  310. // requested resolution. This can happen when camera is captured at high resolutions like 4k
  311. // but the requested resolution is 180. Since getParameters doesn't give us information about
  312. // the resolutions of the simulcast encodings, we have to rely on our initial config for the
  313. // simulcast streams.
  314. || videoStreamEncodings[idx]?.scaleResolutionDownBy === LD_SCALE_FACTOR;
  315. } else {
  316. // For screenshare, keep the HD layer enabled always and the lower layers only for high fps
  317. // screensharing.
  318. active = videoStreamEncodings[idx].scaleResolutionDownBy === HD_SCALE_FACTOR
  319. || !this._isScreenshareBitrateCapped(localVideoTrack);
  320. }
  321. }
  322. return active;
  323. });
  324. return encodingsState;
  325. }
  326. /**
  327. * Returns the calculated max bitrates that need to be configured on the stream encodings based on the video
  328. * type and other considerations associated with screenshare.
  329. *
  330. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  331. * @param {CodecMimeType} codec - The codec currently in use.
  332. * @param {number} newHeight The resolution requested for the video track.
  333. * @returns {Array<number>}
  334. */
  335. calculateEncodingsBitrates(localVideoTrack, codec, newHeight) {
  336. const desktopShareBitrate = this.pc.options?.videoQuality?.desktopbitrate || DESKTOP_SHARE_RATE;
  337. const encodingsBitrates = this._getVideoStreamEncodings(localVideoTrack.getVideoType(), codec)
  338. .map((encoding, idx) => {
  339. let bitrate = encoding.maxBitrate;
  340. // Single video stream.
  341. if (!this.pc.isSpatialScalabilityOn() || this._isRunningInFullSvcMode(codec)) {
  342. const { maxBitrate } = this._calculateActiveEncodingParams(localVideoTrack, codec, newHeight);
  343. return idx === 0 ? maxBitrate : 0;
  344. }
  345. // Multiple video streams.
  346. if (this._isScreenshareBitrateCapped(localVideoTrack)) {
  347. bitrate = desktopShareBitrate;
  348. }
  349. return bitrate;
  350. });
  351. return encodingsBitrates;
  352. }
  353. /**
  354. * Returns the calculated scalability modes for the video encodings when scalability modes are supported.
  355. *
  356. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  357. * @param {CodecMimeType} codec - The codec currently in use.
  358. * @param {number} maxHeight The resolution requested for the video track.
  359. * @returns {Array<VideoEncoderScalabilityMode> | undefined}
  360. */
  361. calculateEncodingsScalabilityMode(localVideoTrack, codec, maxHeight) {
  362. if (!this.pc.isSpatialScalabilityOn() || !this.codecSettings[codec].scalabilityModeEnabled) {
  363. return;
  364. }
  365. // Default modes for simulcast.
  366. const scalabilityModes = [
  367. VideoEncoderScalabilityMode.L1T3,
  368. VideoEncoderScalabilityMode.L1T3,
  369. VideoEncoderScalabilityMode.L1T3
  370. ];
  371. // Full SVC mode.
  372. if (this._isRunningInFullSvcMode(codec)) {
  373. const { scalabilityMode }
  374. = this._calculateActiveEncodingParams(localVideoTrack, codec, maxHeight);
  375. scalabilityModes[0] = scalabilityMode;
  376. scalabilityModes[1] = undefined;
  377. scalabilityModes[2] = undefined;
  378. return scalabilityModes;
  379. }
  380. return scalabilityModes;
  381. }
  382. /**
  383. * Returns the scale factor that needs to be applied on the local video stream based on the desired resolution
  384. * and the codec in use.
  385. *
  386. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  387. * @param {CodecMimeType} codec - The codec currently in use.
  388. * @param {number} maxHeight The resolution requested for the video track.
  389. * @returns {Array<float>}
  390. */
  391. calculateEncodingsScaleFactor(localVideoTrack, codec, maxHeight) {
  392. if (this.pc.isSpatialScalabilityOn() && this.isRunningInSimulcastMode(codec)) {
  393. return this._getVideoStreamEncodings(localVideoTrack.getVideoType(), codec)
  394. .map(encoding => encoding.scaleResolutionDownBy);
  395. }
  396. // Single video stream.
  397. const { scaleResolutionDownBy }
  398. = this._calculateActiveEncodingParams(localVideoTrack, codec, maxHeight);
  399. return [ scaleResolutionDownBy, undefined, undefined ];
  400. }
  401. /**
  402. * Ensures that the ssrcs associated with a FID ssrc-group appear in the correct order, i.e.,
  403. * the primary ssrc first and the secondary rtx ssrc later. This is important for unified
  404. * plan since we have only one FID group per media description.
  405. * @param {Object} description the webRTC session description instance for the remote
  406. * description.
  407. * @private
  408. */
  409. ensureCorrectOrderOfSsrcs(description) {
  410. const parsedSdp = transform.parse(description.sdp);
  411. parsedSdp.media.forEach(mLine => {
  412. if (mLine.type === MediaType.AUDIO) {
  413. return;
  414. }
  415. if (!mLine.ssrcGroups || !mLine.ssrcGroups.length) {
  416. return;
  417. }
  418. let reorderedSsrcs = [];
  419. const ssrcs = new Set();
  420. mLine.ssrcGroups.map(group =>
  421. group.ssrcs
  422. .split(' ')
  423. .filter(Boolean)
  424. .forEach(ssrc => ssrcs.add(ssrc))
  425. );
  426. ssrcs.forEach(ssrc => {
  427. const sources = mLine.ssrcs.filter(source => source.id.toString() === ssrc);
  428. reorderedSsrcs = reorderedSsrcs.concat(sources);
  429. });
  430. mLine.ssrcs = reorderedSsrcs;
  431. });
  432. return new RTCSessionDescription({
  433. type: description.type,
  434. sdp: transform.write(parsedSdp)
  435. });
  436. }
  437. /**
  438. * Returns the max resolution that the client is configured to encode for a given local video track. The actual
  439. * send resolution might be downscaled based on cpu and bandwidth constraints.
  440. *
  441. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  442. * @param {CodecMimeType} codec - The codec currently in use.
  443. * @returns {number|null} The max encoded resolution for the given video track.
  444. */
  445. getConfiguredEncodeResolution(localVideoTrack, codec) {
  446. const height = localVideoTrack.getHeight();
  447. const videoSender = this.pc.findSenderForTrack(localVideoTrack.getTrack());
  448. let maxHeight = 0;
  449. if (!videoSender) {
  450. return null;
  451. }
  452. const parameters = videoSender.getParameters();
  453. if (!parameters?.encodings?.length) {
  454. return null;
  455. }
  456. // SVC mode for VP9 and AV1 codecs.
  457. if (this._isRunningInFullSvcMode(codec)) {
  458. const activeEncoding = parameters.encodings[0];
  459. if (activeEncoding.active) {
  460. return height / activeEncoding.scaleResolutionDownBy;
  461. }
  462. return null;
  463. }
  464. const hasIncorrectConfig = this.pc._capScreenshareBitrate
  465. ? parameters.encodings.every(encoding => encoding.active)
  466. : parameters.encodings.some(encoding => !encoding.active);
  467. const videoType = localVideoTrack.getVideoType();
  468. // Check if every encoding is active for screenshare track when low fps screenshare is configured or some
  469. // of the encodings are disabled when high fps screenshare is configured. In both these cases, the track
  470. // encodings need to be reconfigured. This is needed when p2p->jvb switch happens and new sender constraints
  471. // are not received by the client.
  472. if (videoType === VideoType.DESKTOP && hasIncorrectConfig) {
  473. return null;
  474. }
  475. for (const encoding in parameters.encodings) {
  476. if (parameters.encodings[encoding].active) {
  477. const encodingConfig = this._getVideoStreamEncodings(videoType, codec);
  478. const scaleResolutionDownBy
  479. = this.pc.isSpatialScalabilityOn()
  480. ? encodingConfig[encoding].scaleResolutionDownBy
  481. : parameters.encodings[encoding].scaleResolutionDownBy;
  482. maxHeight = Math.max(maxHeight, height / scaleResolutionDownBy);
  483. }
  484. }
  485. return maxHeight;
  486. }
  487. /**
  488. * Takes in a *unified plan* offer and inserts the appropriate parameters for adding simulcast receive support.
  489. * @param {Object} desc - A session description object
  490. * @param {String} desc.type - the type (offer/answer)
  491. * @param {String} desc.sdp - the sdp content
  492. *
  493. * @return {Object} A session description (same format as above) object with its sdp field modified to advertise
  494. * simulcast receive support.
  495. */
  496. insertUnifiedPlanSimulcastReceive(desc) {
  497. // a=simulcast line is not needed on browsers where we SDP munging is used for enabling on simulcast.
  498. // Remove this check when the client switches to RID/MID based simulcast on all browsers.
  499. if (browser.usesSdpMungingForSimulcast()) {
  500. return desc;
  501. }
  502. const rids = [
  503. {
  504. id: SIM_LAYER_1_RID,
  505. direction: 'recv'
  506. },
  507. {
  508. id: SIM_LAYER_2_RID,
  509. direction: 'recv'
  510. },
  511. {
  512. id: SIM_LAYER_3_RID,
  513. direction: 'recv'
  514. }
  515. ];
  516. // Firefox 72 has stopped parsing the legacy rid= parameters in simulcast attributes.
  517. // eslint-disable-next-line max-len
  518. // https://www.fxsitecompat.dev/en-CA/docs/2019/pt-and-rid-in-webrtc-simulcast-attributes-are-no-longer-supported/
  519. const simulcastLine = browser.isFirefox() && browser.isVersionGreaterThan(71)
  520. ? `recv ${SIM_LAYER_RIDS.join(';')}`
  521. : `recv rid=${SIM_LAYER_RIDS.join(';')}`;
  522. const sdp = transform.parse(desc.sdp);
  523. const mLines = sdp.media.filter(m => m.type === MediaType.VIDEO);
  524. const senderMids = Array.from(this.pc._localTrackTransceiverMids.values());
  525. mLines.forEach((mLine, idx) => {
  526. // Make sure the simulcast recv line is only set on video descriptions that are associated with senders.
  527. if (senderMids.find(sender => mLine.mid.toString() === sender.toString()) || idx === 0) {
  528. if (!mLine.simulcast_03 || !mLine.simulcast) {
  529. mLine.rids = rids;
  530. // eslint-disable-next-line camelcase
  531. mLine.simulcast_03 = {
  532. value: simulcastLine
  533. };
  534. }
  535. } else {
  536. mLine.rids = undefined;
  537. mLine.simulcast = undefined;
  538. // eslint-disable-next-line camelcase
  539. mLine.simulcast_03 = undefined;
  540. }
  541. });
  542. return new RTCSessionDescription({
  543. type: desc.type,
  544. sdp: transform.write(sdp)
  545. });
  546. }
  547. /**
  548. * Returns a boolean indicating whether the video encoder is running in Simulcast mode, i.e., three encodings need
  549. * to be configured in 4:2:1 resolution order with temporal scalability.
  550. *
  551. * @param {CodecMimeType} codec - The video codec in use.
  552. * @returns {boolean}
  553. */
  554. isRunningInSimulcastMode(codec) {
  555. return codec === CodecMimeType.VP8 // VP8 always
  556. // K-SVC mode for VP9 when no scalability mode is set. Though only one outbound-rtp stream is present,
  557. // three separate encodings have to be configured.
  558. || (!this.codecSettings[codec].scalabilityModeEnabled && codec === CodecMimeType.VP9)
  559. // When scalability is enabled, always for H.264, and only when simulcast is explicitly enabled via
  560. // config.js for VP9 and AV1 since full SVC is the default mode for these 2 codecs.
  561. || (this.codecSettings[codec].scalabilityModeEnabled
  562. && (codec === CodecMimeType.H264 || this.codecSettings[codec].useSimulcast));
  563. }
  564. /**
  565. * Replaces the existing track on a RTCRtpSender with the given track.
  566. *
  567. * @param {JitsiLocalTrack} oldTrack - existing track on the sender that needs to be removed.
  568. * @param {JitsiLocalTrack} newTrack - new track that needs to be added to the sender.
  569. * @returns {Promise<RTCRtpTransceiver>} - resolved with the associated transceiver when done, rejected otherwise.
  570. */
  571. replaceTrack(oldTrack, newTrack) {
  572. const mediaType = newTrack?.getType() ?? oldTrack?.getType();
  573. const localTracks = this.pc.getLocalTracks(mediaType);
  574. const track = newTrack?.getTrack() ?? null;
  575. const isNewLocalSource = localTracks?.length
  576. && !oldTrack
  577. && newTrack
  578. && !localTracks.find(t => t === newTrack);
  579. let transceiver;
  580. // If old track exists, replace the track on the corresponding sender.
  581. if (oldTrack && !oldTrack.isMuted()) {
  582. transceiver = this.pc.peerconnection.getTransceivers().find(t => t.sender.track === oldTrack.getTrack());
  583. // Find the first recvonly transceiver when more than one track of the same media type is being added to the pc.
  584. // As part of the track addition, a new m-line was added to the remote description with direction set to
  585. // recvonly.
  586. } else if (isNewLocalSource) {
  587. transceiver = this.pc.peerconnection.getTransceivers().find(
  588. t => t.receiver.track.kind === mediaType
  589. && t.direction === MediaDirection.RECVONLY
  590. // Re-use any existing recvonly transceiver (if available) for p2p case.
  591. && ((this.pc.isP2P && t.currentDirection === MediaDirection.RECVONLY)
  592. || (t.currentDirection === MediaDirection.INACTIVE && !t.stopped)));
  593. // For mute/unmute operations, find the transceiver based on the track index in the source name if present,
  594. // otherwise it is assumed to be the first local track that was added to the peerconnection.
  595. } else {
  596. transceiver = this.pc.peerconnection.getTransceivers().find(t => t.receiver.track.kind === mediaType);
  597. const sourceName = newTrack?.getSourceName() ?? oldTrack?.getSourceName();
  598. if (sourceName) {
  599. const trackIndex = getSourceIndexFromSourceName(sourceName);
  600. if (this.pc.isP2P) {
  601. transceiver = this.pc.peerconnection.getTransceivers()
  602. .filter(t => t.receiver.track.kind === mediaType)[trackIndex];
  603. } else if (oldTrack) {
  604. const transceiverMid = this.pc._localTrackTransceiverMids.get(oldTrack.rtcId);
  605. transceiver = this.pc.peerconnection.getTransceivers().find(t => t.mid === transceiverMid);
  606. } else if (trackIndex) {
  607. transceiver = this.pc.peerconnection.getTransceivers()
  608. .filter(t => t.receiver.track.kind === mediaType
  609. && t.direction !== MediaDirection.RECVONLY)[trackIndex];
  610. }
  611. }
  612. }
  613. if (!transceiver) {
  614. return Promise.reject(
  615. new Error(`Replace track failed - no transceiver for old: ${oldTrack}, new: ${newTrack}`));
  616. }
  617. logger.debug(`${this.pc} Replacing ${oldTrack} with ${newTrack}`);
  618. return transceiver.sender.replaceTrack(track)
  619. .then(() => Promise.resolve(transceiver));
  620. }
  621. /**
  622. * Set the simulcast stream encoding properties on the RTCRtpSender.
  623. *
  624. * @param {JitsiLocalTrack} localTrack - the current track in use for which the encodings are to be set.
  625. * @returns {Promise<void>} - resolved when done.
  626. */
  627. setEncodings(localTrack) {
  628. const mediaType = localTrack.getType();
  629. const transceiver = localTrack?.track && localTrack.getOriginalStream()
  630. ? this.pc.peerconnection.getTransceivers().find(t => t.sender?.track?.id === localTrack.getTrackId())
  631. : this.pc.peerconnection.getTransceivers().find(t => t.receiver?.track?.kind === mediaType);
  632. const parameters = transceiver?.sender?.getParameters();
  633. // Resolve if the encodings are not available yet. This happens immediately after the track is added to the
  634. // peerconnection on chrome in unified-plan. It is ok to ignore and not report the error here since the
  635. // action that triggers 'addTrack' (like unmute) will also configure the encodings and set bitrates after that.
  636. if (!parameters?.encodings?.length) {
  637. return Promise.resolve();
  638. }
  639. parameters.encodings = this._getStreamEncodings(localTrack);
  640. if (mediaType === MediaType.VIDEO) {
  641. return this.pc._updateVideoSenderParameters(() => transceiver.sender.setParameters(parameters));
  642. }
  643. return transceiver.sender.setParameters(parameters);
  644. }
  645. /**
  646. * Resumes or suspends media on the peerconnection by setting the active state on RTCRtpEncodingParameters
  647. * associated with all the senders that have a track attached to it.
  648. *
  649. * @param {boolean} enable - whether outgoing media needs to be enabled or disabled.
  650. * @param {string} mediaType - media type, 'audio' or 'video', if neither is passed, all outgoing media will either
  651. * be enabled or disabled.
  652. * @returns {Promise} - A promise that is resolved when the change is succesful on all the senders, rejected
  653. * otherwise.
  654. */
  655. setMediaTransferActive(enable, mediaType) {
  656. logger.info(`${this.pc} ${enable ? 'Resuming' : 'Suspending'} media transfer.`);
  657. const senders = this.pc.peerconnection.getSenders()
  658. .filter(s => Boolean(s.track) && (!mediaType || s.track.kind === mediaType));
  659. const promises = [];
  660. for (const sender of senders) {
  661. if (sender.track.kind === MediaType.VIDEO) {
  662. promises.push(this.pc._updateVideoSenderParameters(() => this._updateSenderEncodings(sender, enable)));
  663. } else {
  664. promises.push(this._updateSenderEncodings(sender, enable));
  665. }
  666. }
  667. return Promise.allSettled(promises)
  668. .then(settledResult => {
  669. const errors = settledResult
  670. .filter(result => result.status === 'rejected')
  671. .map(result => result.reason);
  672. if (errors.length) {
  673. return Promise.reject(new Error('Failed to change encodings on the RTCRtpSenders'
  674. + `${errors.join(' ')}`));
  675. }
  676. return Promise.resolve();
  677. });
  678. }
  679. /**
  680. * Ensures that the resolution of the stream encodings are consistent with the values
  681. * that were configured on the RTCRtpSender when the source was added to the peerconnection.
  682. * This should prevent us from overriding the default values if the browser returns
  683. * erroneous values when RTCRtpSender.getParameters is used for getting the encodings info.
  684. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  685. * @param {Object} parameters - the RTCRtpEncodingParameters obtained from the browser.
  686. * @returns {void}
  687. */
  688. updateEncodingsResolution(localVideoTrack, parameters) {
  689. if (!(browser.isWebKitBased() && parameters.encodings && Array.isArray(parameters.encodings))) {
  690. return;
  691. }
  692. const allEqualEncodings
  693. = encodings => encodings.every(encoding => typeof encoding.scaleResolutionDownBy !== 'undefined'
  694. && encoding.scaleResolutionDownBy === encodings[0].scaleResolutionDownBy);
  695. // Implement the workaround only when all the encodings report the same resolution.
  696. if (allEqualEncodings(parameters.encodings)) {
  697. const videoStreamEncodings = this._getVideoStreamEncodings(
  698. localVideoTrack.getVideoType(),
  699. this.pc.getConfiguredVideoCodec());
  700. parameters.encodings.forEach((encoding, idx) => {
  701. encoding.scaleResolutionDownBy = videoStreamEncodings[idx].scaleResolutionDownBy;
  702. });
  703. }
  704. }
  705. }