Вы не можете выбрать более 25 тем Темы должны начинаться с буквы или цифры, могут содержать дефисы(-) и должны содержать не более 35 символов.

TPCUtils.js 35KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812
  1. import { getLogger } from '@jitsi/logger';
  2. import clonedeep from 'lodash.clonedeep';
  3. import transform from 'sdp-transform';
  4. import { CodecMimeType } from '../../service/RTC/CodecMimeType';
  5. import { MediaDirection } from '../../service/RTC/MediaDirection';
  6. import { MediaType } from '../../service/RTC/MediaType';
  7. import { getSourceIndexFromSourceName } from '../../service/RTC/SignalingLayer';
  8. import {
  9. SIM_LAYERS,
  10. STANDARD_CODEC_SETTINGS,
  11. VIDEO_QUALITY_LEVELS,
  12. VIDEO_QUALITY_SETTINGS
  13. } from '../../service/RTC/StandardVideoSettings';
  14. import { VideoEncoderScalabilityMode } from '../../service/RTC/VideoEncoderScalabilityMode';
  15. import { VideoType } from '../../service/RTC/VideoType';
  16. import browser from '../browser';
  17. const logger = getLogger(__filename);
  18. const VIDEO_CODECS = [ CodecMimeType.AV1, CodecMimeType.H264, CodecMimeType.VP8, CodecMimeType.VP9 ];
  19. /**
  20. * Handles track related operations on TraceablePeerConnection when browser is
  21. * running in unified plan mode.
  22. */
  23. export class TPCUtils {
  24. /**
  25. * Creates a new instance for a given TraceablePeerConnection
  26. *
  27. * @param peerconnection - the tpc instance for which we have utility functions.
  28. */
  29. constructor(peerconnection) {
  30. this.pc = peerconnection;
  31. this.codecSettings = clonedeep(STANDARD_CODEC_SETTINGS);
  32. const videoQualitySettings = this.pc.options?.videoQuality;
  33. if (videoQualitySettings) {
  34. for (const codec of VIDEO_CODECS) {
  35. const codecConfig = videoQualitySettings[codec];
  36. const bitrateSettings = codecConfig?.maxBitratesVideo
  37. // Read the deprecated settings for max bitrates.
  38. ?? (videoQualitySettings.maxbitratesvideo
  39. && videoQualitySettings.maxbitratesvideo[codec.toUpperCase()]);
  40. if (bitrateSettings) {
  41. const settings = Object.values(VIDEO_QUALITY_SETTINGS);
  42. [ ...settings, 'ssHigh' ].forEach(value => {
  43. if (bitrateSettings[value]) {
  44. this.codecSettings[codec].maxBitratesVideo[value] = bitrateSettings[value];
  45. }
  46. });
  47. }
  48. if (!codecConfig) {
  49. continue; // eslint-disable-line no-continue
  50. }
  51. const scalabilityModeEnabled = this.codecSettings[codec].scalabilityModeEnabled
  52. && (typeof codecConfig.scalabilityModeEnabled === 'undefined'
  53. || codecConfig.scalabilityModeEnabled);
  54. if (scalabilityModeEnabled) {
  55. typeof codecConfig.useSimulcast !== 'undefined'
  56. && (this.codecSettings[codec].useSimulcast = codecConfig.useSimulcast);
  57. typeof codecConfig.useKSVC !== 'undefined'
  58. && (this.codecSettings[codec].useKSVC = codecConfig.useKSVC);
  59. } else {
  60. this.codecSettings[codec].scalabilityModeEnabled = false;
  61. }
  62. }
  63. }
  64. }
  65. /**
  66. * Calculates the configuration of the active encoding when the browser sends only one stream, i,e,, when there is
  67. * no spatial scalability configure (p2p) or when it is running in full SVC mode.
  68. *
  69. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  70. * @param {CodecMimeType} codec - The video codec.
  71. * @param {number} newHeight - The resolution that needs to be configured for the local video track.
  72. * @returns {Object} configuration.
  73. */
  74. _calculateActiveEncodingParams(localVideoTrack, codec, newHeight) {
  75. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  76. const trackCaptureHeight = localVideoTrack.getCaptureResolution();
  77. const effectiveNewHeight = newHeight > trackCaptureHeight ? trackCaptureHeight : newHeight;
  78. const desktopShareBitrate = this.pc.options?.videoQuality?.desktopbitrate || codecBitrates.ssHigh;
  79. const isScreenshare = localVideoTrack.getVideoType() === VideoType.DESKTOP;
  80. let scalabilityMode = this.codecSettings[codec].useKSVC
  81. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3;
  82. const { height, level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= effectiveNewHeight);
  83. let maxBitrate;
  84. let scaleResolutionDownBy = SIM_LAYERS[2].scaleFactor;
  85. if (this._isScreenshareBitrateCapped(localVideoTrack)) {
  86. scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  87. maxBitrate = desktopShareBitrate;
  88. } else if (isScreenshare) {
  89. maxBitrate = codecBitrates.ssHigh;
  90. } else {
  91. maxBitrate = codecBitrates[level];
  92. effectiveNewHeight && (scaleResolutionDownBy = trackCaptureHeight / effectiveNewHeight);
  93. if (height !== effectiveNewHeight) {
  94. logger.debug(`Quality level with height=${height} was picked when requested height=${newHeight} for`
  95. + `track with capture height=${trackCaptureHeight}`);
  96. }
  97. }
  98. const config = {
  99. active: effectiveNewHeight > 0,
  100. maxBitrate,
  101. scalabilityMode,
  102. scaleResolutionDownBy
  103. };
  104. if (!config.active || isScreenshare) {
  105. return config;
  106. }
  107. // Configure the sender to send all 3 spatial layers for resolutions 720p and higher.
  108. switch (level) {
  109. case VIDEO_QUALITY_SETTINGS.ULTRA:
  110. case VIDEO_QUALITY_SETTINGS.FULL:
  111. case VIDEO_QUALITY_SETTINGS.HIGH:
  112. config.scalabilityMode = this.codecSettings[codec].useKSVC
  113. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3;
  114. break;
  115. case VIDEO_QUALITY_SETTINGS.STANDARD:
  116. config.scalabilityMode = this.codecSettings[codec].useKSVC
  117. ? VideoEncoderScalabilityMode.L2T3_KEY : VideoEncoderScalabilityMode.L2T3;
  118. break;
  119. default:
  120. config.scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  121. }
  122. return config;
  123. }
  124. /**
  125. * Obtains stream encodings that need to be configured on the given track based
  126. * on the track media type and the simulcast setting.
  127. * @param {JitsiLocalTrack} localTrack
  128. */
  129. _getStreamEncodings(localTrack) {
  130. if (localTrack.isAudioTrack()) {
  131. return [ { active: this.pc.audioTransferActive } ];
  132. }
  133. const codec = this.pc.getConfiguredVideoCodec();
  134. if (this.pc.isSpatialScalabilityOn()) {
  135. return this._getVideoStreamEncodings(localTrack, codec);
  136. }
  137. return [ {
  138. active: this.pc.videoTransferActive,
  139. maxBitrate: this.codecSettings[codec].maxBitratesVideo.high
  140. } ];
  141. }
  142. /**
  143. * The startup configuration for the stream encodings that are applicable to
  144. * the video stream when a new sender is created on the peerconnection. The initial
  145. * config takes into account the differences in browser's simulcast implementation.
  146. *
  147. * Encoding parameters:
  148. * active - determine the on/off state of a particular encoding.
  149. * maxBitrate - max. bitrate value to be applied to that particular encoding
  150. * based on the encoding's resolution and config.js videoQuality settings if applicable.
  151. * rid - Rtp Stream ID that is configured for a particular simulcast stream.
  152. * scaleResolutionDownBy - the factor by which the encoding is scaled down from the
  153. * original resolution of the captured video.
  154. *
  155. * @param {JitsiLocalTrack} localTrack
  156. * @param {String} codec
  157. */
  158. _getVideoStreamEncodings(localTrack, codec) {
  159. const captureResolution = localTrack.getCaptureResolution();
  160. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  161. const videoType = localTrack.getVideoType();
  162. let effectiveScaleFactors = SIM_LAYERS.map(sim => sim.scaleFactor);
  163. let cameraMaxbitrate;
  164. if (videoType === VideoType.CAMERA) {
  165. const { level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= captureResolution);
  166. cameraMaxbitrate = codecBitrates[level];
  167. if (level === VIDEO_QUALITY_SETTINGS.ULTRA) {
  168. effectiveScaleFactors[1] = 6.0; // 360p
  169. effectiveScaleFactors[0] = 12.0; // 180p
  170. } else if (level === VIDEO_QUALITY_SETTINGS.FULL) {
  171. effectiveScaleFactors[1] = 3.0; // 360p
  172. effectiveScaleFactors[0] = 6.0; // 180p
  173. }
  174. }
  175. const maxBitrate = videoType === VideoType.DESKTOP
  176. ? codecBitrates.ssHigh : cameraMaxbitrate;
  177. let effectiveBitrates = [ codecBitrates.low, codecBitrates.standard, maxBitrate ];
  178. // The SSRCs on older versions of Firefox are reversed in SDP, i.e., they have resolution order of 1:2:4 as
  179. // opposed to Chromium and other browsers. This has been reverted in Firefox 117 as part of the below commit.
  180. // https://hg.mozilla.org/mozilla-central/rev/b0348f1f8d7197fb87158ba74542d28d46133997
  181. // This revert seems to be applied only to camera tracks, the desktop stream encodings still have the
  182. // resolution order of 4:2:1.
  183. if (browser.isFirefox() && (videoType === VideoType.DESKTOP || browser.isVersionLessThan(117))) {
  184. effectiveBitrates = effectiveBitrates.reverse();
  185. effectiveScaleFactors = effectiveScaleFactors.reverse();
  186. }
  187. const standardSimulcastEncodings = [
  188. {
  189. active: this.pc.videoTransferActive,
  190. maxBitrate: effectiveBitrates[0],
  191. rid: SIM_LAYERS[0].rid,
  192. scaleResolutionDownBy: effectiveScaleFactors[0]
  193. },
  194. {
  195. active: this.pc.videoTransferActive,
  196. maxBitrate: effectiveBitrates[1],
  197. rid: SIM_LAYERS[1].rid,
  198. scaleResolutionDownBy: effectiveScaleFactors[1]
  199. },
  200. {
  201. active: this.pc.videoTransferActive,
  202. maxBitrate: effectiveBitrates[2],
  203. rid: SIM_LAYERS[2].rid,
  204. scaleResolutionDownBy: effectiveScaleFactors[2]
  205. }
  206. ];
  207. if (this.codecSettings[codec].scalabilityModeEnabled) {
  208. // Configure all 3 encodings when simulcast is requested through config.js for AV1 and VP9 and for H.264
  209. // always since that is the only supported mode when DD header extension is negotiated for H.264.
  210. if (this.codecSettings[codec].useSimulcast || codec === CodecMimeType.H264) {
  211. for (const encoding of standardSimulcastEncodings) {
  212. encoding.scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  213. }
  214. return standardSimulcastEncodings;
  215. }
  216. // Configure only one encoding for the SVC mode.
  217. return [
  218. {
  219. active: this.pc.videoTransferActive,
  220. maxBitrate: effectiveBitrates[2],
  221. rid: SIM_LAYERS[0].rid,
  222. scaleResolutionDownBy: effectiveScaleFactors[2],
  223. scalabilityMode: this.codecSettings[codec].useKSVC
  224. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3
  225. },
  226. {
  227. active: false,
  228. maxBitrate: 0
  229. },
  230. {
  231. active: false,
  232. maxBitrate: 0
  233. }
  234. ];
  235. }
  236. return standardSimulcastEncodings;
  237. }
  238. /**
  239. * Returns a boolean indicating whether the video encoder is running in full SVC mode, i.e., it sends only one
  240. * video stream that has both temporal and spatial scalability.
  241. *
  242. * @param {CodecMimeType} codec
  243. * @returns boolean
  244. */
  245. _isRunningInFullSvcMode(codec) {
  246. return (codec === CodecMimeType.VP9 || codec === CodecMimeType.AV1)
  247. && this.codecSettings[codec].scalabilityModeEnabled
  248. && !this.codecSettings[codec].useSimulcast;
  249. }
  250. /**
  251. * Returns a boolean indicating whether the bitrate needs to be capped for the local video track if it happens to
  252. * be a screenshare track. The lower spatial layers for screensharing are disabled when low fps screensharing is in
  253. * progress. Sending all three streams often results in the browser suspending the high resolution in low b/w and
  254. * and low cpu conditions, especially on the low end machines. Suspending the low resolution streams ensures that
  255. * the highest resolution stream is available always. Safari is an exception here since it does not send the
  256. * desktop stream at all if only the high resolution stream is enabled.
  257. *
  258. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  259. * @returns {boolean}
  260. */
  261. _isScreenshareBitrateCapped(localVideoTrack) {
  262. return localVideoTrack.getVideoType() === VideoType.DESKTOP
  263. && this.pc._capScreenshareBitrate
  264. && !browser.isWebKitBased();
  265. }
  266. /**
  267. * Updates the sender parameters in the stream encodings.
  268. *
  269. * @param {RTCRtpSender} sender - the sender associated with a MediaStreamTrack.
  270. * @param {boolean} enable - whether the streams needs to be enabled or disabled.
  271. * @returns {Promise} - A promise that resolves when the operation is successful, rejected otherwise.
  272. */
  273. _updateSenderEncodings(sender, enable) {
  274. const parameters = sender.getParameters();
  275. if (parameters?.encodings?.length) {
  276. for (const encoding of parameters.encodings) {
  277. encoding.active = enable;
  278. }
  279. }
  280. return sender.setParameters(parameters);
  281. }
  282. /**
  283. * Adds {@link JitsiLocalTrack} to the WebRTC peerconnection for the first time.
  284. * @param {JitsiLocalTrack} track - track to be added to the peerconnection.
  285. * @param {boolean} isInitiator - boolean that indicates if the endpoint is offerer in a p2p connection.
  286. * @returns {void}
  287. */
  288. addTrack(localTrack, isInitiator) {
  289. const track = localTrack.getTrack();
  290. if (isInitiator) {
  291. const streams = [];
  292. if (localTrack.getOriginalStream()) {
  293. streams.push(localTrack.getOriginalStream());
  294. }
  295. // Use pc.addTransceiver() for the initiator case when local tracks are getting added
  296. // to the peerconnection before a session-initiate is sent over to the peer.
  297. const transceiverInit = {
  298. direction: MediaDirection.SENDRECV,
  299. streams,
  300. sendEncodings: []
  301. };
  302. if (!browser.isFirefox()) {
  303. transceiverInit.sendEncodings = this._getStreamEncodings(localTrack);
  304. }
  305. this.pc.peerconnection.addTransceiver(track, transceiverInit);
  306. } else {
  307. // Use pc.addTrack() for responder case so that we can re-use the m-lines that were created
  308. // when setRemoteDescription was called. pc.addTrack() automatically attaches to any existing
  309. // unused "recv-only" transceiver.
  310. this.pc.peerconnection.addTrack(track);
  311. }
  312. }
  313. /**
  314. * Returns the calculated active state of the stream encodings based on the frame height requested for the send
  315. * stream. All the encodings that have a resolution lower than the frame height requested will be enabled.
  316. *
  317. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  318. * @param {CodecMimeType} codec - The codec currently in use.
  319. * @param {number} newHeight The resolution requested for the video track.
  320. * @returns {Array<boolean>}
  321. */
  322. calculateEncodingsActiveState(localVideoTrack, codec, newHeight) {
  323. const height = localVideoTrack.getCaptureResolution();
  324. const videoStreamEncodings = this._getVideoStreamEncodings(localVideoTrack, codec);
  325. const encodingsState = videoStreamEncodings
  326. .map(encoding => height / encoding.scaleResolutionDownBy)
  327. .map((frameHeight, idx) => {
  328. // Single video stream.
  329. if (!this.pc.isSpatialScalabilityOn() || this._isRunningInFullSvcMode(codec)) {
  330. const { active } = this._calculateActiveEncodingParams(localVideoTrack, codec, newHeight);
  331. return idx === 0 ? active : false;
  332. }
  333. // Multiple video streams.
  334. let active = false;
  335. if (newHeight > 0) {
  336. if (localVideoTrack.getVideoType() === VideoType.CAMERA) {
  337. active = frameHeight <= newHeight
  338. // Keep the LD stream enabled even when the LD stream's resolution is higher than of the
  339. // requested resolution. This can happen when camera is captured at high resolutions like 4k
  340. // but the requested resolution is 180. Since getParameters doesn't give us information about
  341. // the resolutions of the simulcast encodings, we have to rely on our initial config for the
  342. // simulcast streams.
  343. || videoStreamEncodings[idx]?.scaleResolutionDownBy === SIM_LAYERS[0].scaleFactor;
  344. } else {
  345. // For screenshare, keep the HD layer enabled always and the lower layers only for high fps
  346. // screensharing.
  347. active = videoStreamEncodings[idx].scaleResolutionDownBy === SIM_LAYERS[2].scaleFactor
  348. || !this._isScreenshareBitrateCapped(localVideoTrack);
  349. }
  350. }
  351. return active;
  352. });
  353. return encodingsState;
  354. }
  355. /**
  356. * Returns the calculated max bitrates that need to be configured on the stream encodings based on the video
  357. * type and other considerations associated with screenshare.
  358. *
  359. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  360. * @param {CodecMimeType} codec - The codec currently in use.
  361. * @param {number} newHeight The resolution requested for the video track.
  362. * @returns {Array<number>}
  363. */
  364. calculateEncodingsBitrates(localVideoTrack, codec, newHeight) {
  365. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  366. const desktopShareBitrate = this.pc.options?.videoQuality?.desktopbitrate || codecBitrates.ssHigh;
  367. const encodingsBitrates = this._getVideoStreamEncodings(localVideoTrack, codec)
  368. .map((encoding, idx) => {
  369. let bitrate = encoding.maxBitrate;
  370. // Single video stream.
  371. if (!this.pc.isSpatialScalabilityOn() || this._isRunningInFullSvcMode(codec)) {
  372. const { maxBitrate } = this._calculateActiveEncodingParams(localVideoTrack, codec, newHeight);
  373. return idx === 0 ? maxBitrate : 0;
  374. }
  375. // Multiple video streams.
  376. if (this._isScreenshareBitrateCapped(localVideoTrack)) {
  377. bitrate = desktopShareBitrate;
  378. }
  379. return bitrate;
  380. });
  381. return encodingsBitrates;
  382. }
  383. /**
  384. * Returns the calculated scalability modes for the video encodings when scalability modes are supported.
  385. *
  386. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  387. * @param {CodecMimeType} codec - The codec currently in use.
  388. * @param {number} maxHeight The resolution requested for the video track.
  389. * @returns {Array<VideoEncoderScalabilityMode> | undefined}
  390. */
  391. calculateEncodingsScalabilityMode(localVideoTrack, codec, maxHeight) {
  392. if (!this.pc.isSpatialScalabilityOn() || !this.codecSettings[codec].scalabilityModeEnabled) {
  393. return;
  394. }
  395. // Default modes for simulcast.
  396. const scalabilityModes = [
  397. VideoEncoderScalabilityMode.L1T3,
  398. VideoEncoderScalabilityMode.L1T3,
  399. VideoEncoderScalabilityMode.L1T3
  400. ];
  401. // Full SVC mode.
  402. if (this._isRunningInFullSvcMode(codec)) {
  403. const { scalabilityMode }
  404. = this._calculateActiveEncodingParams(localVideoTrack, codec, maxHeight);
  405. scalabilityModes[0] = scalabilityMode;
  406. scalabilityModes[1] = undefined;
  407. scalabilityModes[2] = undefined;
  408. return scalabilityModes;
  409. }
  410. return scalabilityModes;
  411. }
  412. /**
  413. * Returns the scale factor that needs to be applied on the local video stream based on the desired resolution
  414. * and the codec in use.
  415. *
  416. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  417. * @param {CodecMimeType} codec - The codec currently in use.
  418. * @param {number} maxHeight The resolution requested for the video track.
  419. * @returns {Array<float>}
  420. */
  421. calculateEncodingsScaleFactor(localVideoTrack, codec, maxHeight) {
  422. if (this.pc.isSpatialScalabilityOn() && this.isRunningInSimulcastMode(codec)) {
  423. return this._getVideoStreamEncodings(localVideoTrack, codec)
  424. .map(encoding => encoding.scaleResolutionDownBy);
  425. }
  426. // Single video stream.
  427. const { scaleResolutionDownBy }
  428. = this._calculateActiveEncodingParams(localVideoTrack, codec, maxHeight);
  429. return [ scaleResolutionDownBy, undefined, undefined ];
  430. }
  431. /**
  432. * Ensures that the ssrcs associated with a FID ssrc-group appear in the correct order, i.e.,
  433. * the primary ssrc first and the secondary rtx ssrc later. This is important for unified
  434. * plan since we have only one FID group per media description.
  435. * @param {Object} description the webRTC session description instance for the remote
  436. * description.
  437. * @private
  438. */
  439. ensureCorrectOrderOfSsrcs(description) {
  440. const parsedSdp = transform.parse(description.sdp);
  441. parsedSdp.media.forEach(mLine => {
  442. if (mLine.type === MediaType.AUDIO) {
  443. return;
  444. }
  445. if (!mLine.ssrcGroups || !mLine.ssrcGroups.length) {
  446. return;
  447. }
  448. let reorderedSsrcs = [];
  449. const ssrcs = new Set();
  450. mLine.ssrcGroups.map(group =>
  451. group.ssrcs
  452. .split(' ')
  453. .filter(Boolean)
  454. .forEach(ssrc => ssrcs.add(ssrc))
  455. );
  456. ssrcs.forEach(ssrc => {
  457. const sources = mLine.ssrcs.filter(source => source.id.toString() === ssrc);
  458. reorderedSsrcs = reorderedSsrcs.concat(sources);
  459. });
  460. mLine.ssrcs = reorderedSsrcs;
  461. });
  462. return new RTCSessionDescription({
  463. type: description.type,
  464. sdp: transform.write(parsedSdp)
  465. });
  466. }
  467. /**
  468. * Returns the max resolution that the client is configured to encode for a given local video track. The actual
  469. * send resolution might be downscaled based on cpu and bandwidth constraints.
  470. *
  471. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  472. * @param {CodecMimeType} codec - The codec currently in use.
  473. * @returns {number|null} The max encoded resolution for the given video track.
  474. */
  475. getConfiguredEncodeResolution(localVideoTrack, codec) {
  476. const height = localVideoTrack.getCaptureResolution();
  477. const videoSender = this.pc.findSenderForTrack(localVideoTrack.getTrack());
  478. let maxHeight = 0;
  479. if (!videoSender) {
  480. return null;
  481. }
  482. const parameters = videoSender.getParameters();
  483. if (!parameters?.encodings?.length) {
  484. return null;
  485. }
  486. // SVC mode for VP9 and AV1 codecs.
  487. if (this._isRunningInFullSvcMode(codec)) {
  488. const activeEncoding = parameters.encodings[0];
  489. if (activeEncoding.active) {
  490. return height / activeEncoding.scaleResolutionDownBy;
  491. }
  492. return null;
  493. }
  494. const hasIncorrectConfig = this.pc._capScreenshareBitrate
  495. ? parameters.encodings.every(encoding => encoding.active)
  496. : parameters.encodings.some(encoding => !encoding.active);
  497. // Check if every encoding is active for screenshare track when low fps screenshare is configured or some
  498. // of the encodings are disabled when high fps screenshare is configured. In both these cases, the track
  499. // encodings need to be reconfigured. This is needed when p2p->jvb switch happens and new sender constraints
  500. // are not received by the client.
  501. if (localVideoTrack.getVideoType() === VideoType.DESKTOP && hasIncorrectConfig) {
  502. return null;
  503. }
  504. for (const encoding in parameters.encodings) {
  505. if (parameters.encodings[encoding].active) {
  506. const encodingConfig = this._getVideoStreamEncodings(localVideoTrack, codec);
  507. const scaleResolutionDownBy
  508. = this.pc.isSpatialScalabilityOn()
  509. ? encodingConfig[encoding].scaleResolutionDownBy
  510. : parameters.encodings[encoding].scaleResolutionDownBy;
  511. maxHeight = Math.max(maxHeight, height / scaleResolutionDownBy);
  512. }
  513. }
  514. return maxHeight;
  515. }
  516. /**
  517. * Takes in a *unified plan* offer and inserts the appropriate parameters for adding simulcast receive support.
  518. * @param {Object} desc - A session description object
  519. * @param {String} desc.type - the type (offer/answer)
  520. * @param {String} desc.sdp - the sdp content
  521. *
  522. * @return {Object} A session description (same format as above) object with its sdp field modified to advertise
  523. * simulcast receive support.
  524. */
  525. insertUnifiedPlanSimulcastReceive(desc) {
  526. // a=simulcast line is not needed on browsers where we SDP munging is used for enabling on simulcast.
  527. // Remove this check when the client switches to RID/MID based simulcast on all browsers.
  528. if (browser.usesSdpMungingForSimulcast()) {
  529. return desc;
  530. }
  531. const rids = [
  532. {
  533. id: SIM_LAYERS[0].rid,
  534. direction: 'recv'
  535. },
  536. {
  537. id: SIM_LAYERS[1].rid,
  538. direction: 'recv'
  539. },
  540. {
  541. id: SIM_LAYERS[2].rid,
  542. direction: 'recv'
  543. }
  544. ];
  545. const ridLine = rids.map(val => val.id).join(';');
  546. const simulcastLine = `recv ${ridLine}`;
  547. const sdp = transform.parse(desc.sdp);
  548. const mLines = sdp.media.filter(m => m.type === MediaType.VIDEO);
  549. const senderMids = Array.from(this.pc._localTrackTransceiverMids.values());
  550. mLines.forEach((mLine, idx) => {
  551. // Make sure the simulcast recv line is only set on video descriptions that are associated with senders.
  552. if (senderMids.find(sender => mLine.mid.toString() === sender.toString()) || idx === 0) {
  553. if (!mLine.simulcast_03 || !mLine.simulcast) {
  554. mLine.rids = rids;
  555. // eslint-disable-next-line camelcase
  556. mLine.simulcast_03 = {
  557. value: simulcastLine
  558. };
  559. }
  560. } else {
  561. mLine.rids = undefined;
  562. mLine.simulcast = undefined;
  563. // eslint-disable-next-line camelcase
  564. mLine.simulcast_03 = undefined;
  565. }
  566. });
  567. return new RTCSessionDescription({
  568. type: desc.type,
  569. sdp: transform.write(sdp)
  570. });
  571. }
  572. /**
  573. * Returns a boolean indicating whether the video encoder is running in Simulcast mode, i.e., three encodings need
  574. * to be configured in 4:2:1 resolution order with temporal scalability.
  575. *
  576. * @param {CodecMimeType} codec - The video codec in use.
  577. * @returns {boolean}
  578. */
  579. isRunningInSimulcastMode(codec) {
  580. return codec === CodecMimeType.VP8 // VP8 always
  581. // K-SVC mode for VP9 when no scalability mode is set. Though only one outbound-rtp stream is present,
  582. // three separate encodings have to be configured.
  583. || (!this.codecSettings[codec].scalabilityModeEnabled && codec === CodecMimeType.VP9)
  584. // When scalability is enabled, always for H.264, and only when simulcast is explicitly enabled via
  585. // config.js for VP9 and AV1 since full SVC is the default mode for these 2 codecs.
  586. || (this.codecSettings[codec].scalabilityModeEnabled
  587. && (codec === CodecMimeType.H264 || this.codecSettings[codec].useSimulcast));
  588. }
  589. /**
  590. * Replaces the existing track on a RTCRtpSender with the given track.
  591. *
  592. * @param {JitsiLocalTrack} oldTrack - existing track on the sender that needs to be removed.
  593. * @param {JitsiLocalTrack} newTrack - new track that needs to be added to the sender.
  594. * @returns {Promise<RTCRtpTransceiver>} - resolved with the associated transceiver when done, rejected otherwise.
  595. */
  596. replaceTrack(oldTrack, newTrack) {
  597. const mediaType = newTrack?.getType() ?? oldTrack?.getType();
  598. const localTracks = this.pc.getLocalTracks(mediaType);
  599. const track = newTrack?.getTrack() ?? null;
  600. const isNewLocalSource = localTracks?.length
  601. && !oldTrack
  602. && newTrack
  603. && !localTracks.find(t => t === newTrack);
  604. let transceiver;
  605. // If old track exists, replace the track on the corresponding sender.
  606. if (oldTrack && !oldTrack.isMuted()) {
  607. transceiver = this.pc.peerconnection.getTransceivers().find(t => t.sender.track === oldTrack.getTrack());
  608. // Find the first recvonly transceiver when more than one track of the same media type is being added to the pc.
  609. // As part of the track addition, a new m-line was added to the remote description with direction set to
  610. // recvonly.
  611. } else if (isNewLocalSource) {
  612. transceiver = this.pc.peerconnection.getTransceivers().find(
  613. t => t.receiver.track.kind === mediaType
  614. && t.direction === MediaDirection.RECVONLY
  615. // Re-use any existing recvonly transceiver (if available) for p2p case.
  616. && ((this.pc.isP2P && t.currentDirection === MediaDirection.RECVONLY)
  617. || (t.currentDirection === MediaDirection.INACTIVE && !t.stopped)));
  618. // For mute/unmute operations, find the transceiver based on the track index in the source name if present,
  619. // otherwise it is assumed to be the first local track that was added to the peerconnection.
  620. } else {
  621. transceiver = this.pc.peerconnection.getTransceivers().find(t => t.receiver.track.kind === mediaType);
  622. const sourceName = newTrack?.getSourceName() ?? oldTrack?.getSourceName();
  623. if (sourceName) {
  624. const trackIndex = getSourceIndexFromSourceName(sourceName);
  625. if (this.pc.isP2P) {
  626. transceiver = this.pc.peerconnection.getTransceivers()
  627. .filter(t => t.receiver.track.kind === mediaType)[trackIndex];
  628. } else if (oldTrack) {
  629. const transceiverMid = this.pc._localTrackTransceiverMids.get(oldTrack.rtcId);
  630. transceiver = this.pc.peerconnection.getTransceivers().find(t => t.mid === transceiverMid);
  631. } else if (trackIndex) {
  632. transceiver = this.pc.peerconnection.getTransceivers()
  633. .filter(t => t.receiver.track.kind === mediaType
  634. && t.direction !== MediaDirection.RECVONLY)[trackIndex];
  635. }
  636. }
  637. }
  638. if (!transceiver) {
  639. return Promise.reject(
  640. new Error(`Replace track failed - no transceiver for old: ${oldTrack}, new: ${newTrack}`));
  641. }
  642. logger.debug(`${this.pc} Replacing ${oldTrack} with ${newTrack}`);
  643. return transceiver.sender.replaceTrack(track)
  644. .then(() => Promise.resolve(transceiver));
  645. }
  646. /**
  647. * Set the simulcast stream encoding properties on the RTCRtpSender.
  648. *
  649. * @param {JitsiLocalTrack} localTrack - the current track in use for which the encodings are to be set.
  650. * @returns {Promise<void>} - resolved when done.
  651. */
  652. setEncodings(localTrack) {
  653. const mediaType = localTrack.getType();
  654. const transceiver = localTrack?.track && localTrack.getOriginalStream()
  655. ? this.pc.peerconnection.getTransceivers().find(t => t.sender?.track?.id === localTrack.getTrackId())
  656. : this.pc.peerconnection.getTransceivers().find(t => t.receiver?.track?.kind === mediaType);
  657. const parameters = transceiver?.sender?.getParameters();
  658. // Resolve if the encodings are not available yet. This happens immediately after the track is added to the
  659. // peerconnection on chrome in unified-plan. It is ok to ignore and not report the error here since the
  660. // action that triggers 'addTrack' (like unmute) will also configure the encodings and set bitrates after that.
  661. if (!parameters?.encodings?.length) {
  662. return Promise.resolve();
  663. }
  664. parameters.encodings = this._getStreamEncodings(localTrack);
  665. if (mediaType === MediaType.VIDEO) {
  666. return this.pc._updateVideoSenderParameters(() => transceiver.sender.setParameters(parameters));
  667. }
  668. return transceiver.sender.setParameters(parameters);
  669. }
  670. /**
  671. * Resumes or suspends media on the peerconnection by setting the active state on RTCRtpEncodingParameters
  672. * associated with all the senders that have a track attached to it.
  673. *
  674. * @param {boolean} enable - whether outgoing media needs to be enabled or disabled.
  675. * @param {string} mediaType - media type, 'audio' or 'video', if neither is passed, all outgoing media will either
  676. * be enabled or disabled.
  677. * @returns {Promise} - A promise that is resolved when the change is succesful on all the senders, rejected
  678. * otherwise.
  679. */
  680. setMediaTransferActive(enable, mediaType) {
  681. logger.info(`${this.pc} ${enable ? 'Resuming' : 'Suspending'} media transfer.`);
  682. const senders = this.pc.peerconnection.getSenders()
  683. .filter(s => Boolean(s.track) && (!mediaType || s.track.kind === mediaType));
  684. const promises = [];
  685. for (const sender of senders) {
  686. if (sender.track.kind === MediaType.VIDEO) {
  687. promises.push(this.pc._updateVideoSenderParameters(() => this._updateSenderEncodings(sender, enable)));
  688. } else {
  689. promises.push(this._updateSenderEncodings(sender, enable));
  690. }
  691. }
  692. return Promise.allSettled(promises)
  693. .then(settledResult => {
  694. const errors = settledResult
  695. .filter(result => result.status === 'rejected')
  696. .map(result => result.reason);
  697. if (errors.length) {
  698. return Promise.reject(new Error('Failed to change encodings on the RTCRtpSenders'
  699. + `${errors.join(' ')}`));
  700. }
  701. return Promise.resolve();
  702. });
  703. }
  704. }