Вы не можете выбрать более 25 тем Темы должны начинаться с буквы или цифры, могут содержать дефисы(-) и должны содержать не более 35 символов.

TPCUtils.js 40KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964
  1. import { getLogger } from '@jitsi/logger';
  2. import { cloneDeep } from 'lodash-es';
  3. import transform from 'sdp-transform';
  4. import { CodecMimeType } from '../../service/RTC/CodecMimeType';
  5. import { MediaDirection } from '../../service/RTC/MediaDirection';
  6. import { MediaType } from '../../service/RTC/MediaType';
  7. import {
  8. SIM_LAYERS,
  9. SSRC_GROUP_SEMANTICS,
  10. STANDARD_CODEC_SETTINGS,
  11. VIDEO_QUALITY_LEVELS,
  12. VIDEO_QUALITY_SETTINGS
  13. } from '../../service/RTC/StandardVideoQualitySettings';
  14. import { VideoEncoderScalabilityMode } from '../../service/RTC/VideoEncoderScalabilityMode';
  15. import { VideoType } from '../../service/RTC/VideoType';
  16. import browser from '../browser';
  17. import SDPUtil from '../sdp/SDPUtil';
  18. const logger = getLogger(__filename);
  19. const DD_HEADER_EXT_URI
  20. = 'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension';
  21. const DD_HEADER_EXT_ID = 11;
  22. const VIDEO_CODECS = [ CodecMimeType.AV1, CodecMimeType.H264, CodecMimeType.VP8, CodecMimeType.VP9 ];
  23. /**
  24. * Handles all the utility functions for the TraceablePeerConnection class, like calculating the encoding parameters,
  25. * determining the media direction, calculating bitrates based on the current codec settings, etc.
  26. */
  27. export class TPCUtils {
  28. /**
  29. * Creates a new instance for a given TraceablePeerConnection
  30. *
  31. * @param peerconnection - the tpc instance for which we have utility functions.
  32. * @param options - additional options that can be passed to the utility functions.
  33. * @param options.audioQuality - the audio quality settings that are used to calculate the audio codec parameters.
  34. * @param options.isP2P - whether the connection is a P2P connection.
  35. * @param options.videoQuality - the video quality settings that are used to calculate the encoding parameters.
  36. */
  37. constructor(peerconnection, options = {}) {
  38. this.pc = peerconnection;
  39. this.options = options;
  40. this.codecSettings = cloneDeep(STANDARD_CODEC_SETTINGS);
  41. /**
  42. * Flag indicating bridge support for AV1 codec. On the bridge connection, it is supported only when support for
  43. * Dependency Descriptor header extensions is offered by Jicofo. H.264 simulcast is also possible when these
  44. * header extensions are negotiated.
  45. */
  46. this.supportsDDHeaderExt = false;
  47. /**
  48. * Reads videoQuality settings from config.js and overrides the code defaults for video codecs.
  49. */
  50. const videoQualitySettings = this.options.videoQuality;
  51. if (videoQualitySettings) {
  52. for (const codec of VIDEO_CODECS) {
  53. const codecConfig = videoQualitySettings[codec];
  54. const bitrateSettings = codecConfig?.maxBitratesVideo
  55. // Read the deprecated settings for max bitrates.
  56. ?? (videoQualitySettings.maxbitratesvideo
  57. && videoQualitySettings.maxbitratesvideo[codec.toUpperCase()]);
  58. if (bitrateSettings) {
  59. const settings = Object.values(VIDEO_QUALITY_SETTINGS);
  60. [ ...settings, 'ssHigh' ].forEach(value => {
  61. if (bitrateSettings[value]) {
  62. this.codecSettings[codec].maxBitratesVideo[value] = bitrateSettings[value];
  63. }
  64. });
  65. }
  66. if (!codecConfig) {
  67. continue; // eslint-disable-line no-continue
  68. }
  69. const scalabilityModeEnabled = this.codecSettings[codec].scalabilityModeEnabled
  70. && (typeof codecConfig.scalabilityModeEnabled === 'undefined'
  71. || codecConfig.scalabilityModeEnabled);
  72. if (scalabilityModeEnabled) {
  73. typeof codecConfig.useSimulcast !== 'undefined'
  74. && (this.codecSettings[codec].useSimulcast = codecConfig.useSimulcast);
  75. typeof codecConfig.useKSVC !== 'undefined'
  76. && (this.codecSettings[codec].useKSVC = codecConfig.useKSVC);
  77. } else {
  78. this.codecSettings[codec].scalabilityModeEnabled = false;
  79. }
  80. }
  81. }
  82. }
  83. /**
  84. * Calculates the configuration of the active encoding when the browser sends only one stream, i,e,, when there is
  85. * no spatial scalability configure (p2p) or when it is running in full SVC mode.
  86. *
  87. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  88. * @param {CodecMimeType} codec - The video codec.
  89. * @param {number} newHeight - The resolution that needs to be configured for the local video track.
  90. * @returns {Object} configuration.
  91. * @private
  92. */
  93. _calculateActiveEncodingParams(localVideoTrack, codec, newHeight) {
  94. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  95. const trackCaptureHeight = localVideoTrack.getCaptureResolution();
  96. const effectiveNewHeight = newHeight > trackCaptureHeight ? trackCaptureHeight : newHeight;
  97. const desktopShareBitrate = this.options.videoQuality?.desktopbitrate || codecBitrates.ssHigh;
  98. const isScreenshare = localVideoTrack.getVideoType() === VideoType.DESKTOP;
  99. let scalabilityMode = this.codecSettings[codec].useKSVC
  100. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3;
  101. const { height, level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= effectiveNewHeight);
  102. let maxBitrate;
  103. let scaleResolutionDownBy = SIM_LAYERS[2].scaleFactor;
  104. if (this._isScreenshareBitrateCapped(localVideoTrack)) {
  105. scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  106. maxBitrate = desktopShareBitrate;
  107. } else if (isScreenshare) {
  108. maxBitrate = codecBitrates.ssHigh;
  109. } else {
  110. maxBitrate = codecBitrates[level];
  111. effectiveNewHeight && (scaleResolutionDownBy = trackCaptureHeight / effectiveNewHeight);
  112. if (height !== effectiveNewHeight) {
  113. logger.debug(`Quality level with height=${height} was picked when requested height=${newHeight} for`
  114. + `track with capture height=${trackCaptureHeight}`);
  115. }
  116. }
  117. const config = {
  118. active: effectiveNewHeight > 0,
  119. maxBitrate,
  120. scalabilityMode,
  121. scaleResolutionDownBy
  122. };
  123. if (!config.active || isScreenshare) {
  124. return config;
  125. }
  126. // Configure the sender to send all 3 spatial layers for resolutions 720p and higher.
  127. switch (level) {
  128. case VIDEO_QUALITY_SETTINGS.ULTRA:
  129. case VIDEO_QUALITY_SETTINGS.FULL:
  130. case VIDEO_QUALITY_SETTINGS.HIGH:
  131. config.scalabilityMode = this.codecSettings[codec].useKSVC
  132. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3;
  133. break;
  134. case VIDEO_QUALITY_SETTINGS.STANDARD:
  135. config.scalabilityMode = this.codecSettings[codec].useKSVC
  136. ? VideoEncoderScalabilityMode.L2T3_KEY : VideoEncoderScalabilityMode.L2T3;
  137. break;
  138. default:
  139. config.scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  140. }
  141. return config;
  142. }
  143. /**
  144. * Returns the codecs in the current order of preference in the SDP provided.
  145. *
  146. * @param {transform.SessionDescription} parsedSdp the parsed SDP object.
  147. * @returns {Array<CodecMimeType>}
  148. * @private
  149. */
  150. _getConfiguredVideoCodecsImpl(parsedSdp) {
  151. const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO);
  152. const codecs = new Set(mLine.rtp
  153. .filter(pt => pt.codec.toLowerCase() !== 'rtx')
  154. .map(pt => pt.codec.toLowerCase()));
  155. return Array.from(codecs);
  156. }
  157. /**
  158. * The startup configuration for the stream encodings that are applicable to the video stream when a new sender is
  159. * created on the peerconnection. The initial config takes into account the differences in browser's simulcast
  160. * implementation.
  161. *
  162. * Encoding parameters:
  163. * active - determine the on/off state of a particular encoding.
  164. * maxBitrate - max. bitrate value to be applied to that particular encoding based on the encoding's resolution and
  165. * config.js videoQuality settings if applicable.
  166. * rid - Rtp Stream ID that is configured for a particular simulcast stream.
  167. * scaleResolutionDownBy - the factor by which the encoding is scaled down from the original resolution of the
  168. * captured video.
  169. *
  170. * @param {JitsiLocalTrack} localTrack - The local video track.
  171. * @param {String} codec - The codec currently in use.
  172. * @returns {Array<Object>} - The initial configuration for the stream encodings.
  173. * @private
  174. */
  175. _getVideoStreamEncodings(localTrack, codec) {
  176. const captureResolution = localTrack.getCaptureResolution();
  177. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  178. const videoType = localTrack.getVideoType();
  179. let effectiveScaleFactors = SIM_LAYERS.map(sim => sim.scaleFactor);
  180. let cameraMaxbitrate;
  181. if (videoType === VideoType.CAMERA) {
  182. const { level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= captureResolution);
  183. cameraMaxbitrate = codecBitrates[level];
  184. if (level === VIDEO_QUALITY_SETTINGS.ULTRA) {
  185. effectiveScaleFactors[1] = 6.0; // 360p
  186. effectiveScaleFactors[0] = 12.0; // 180p
  187. } else if (level === VIDEO_QUALITY_SETTINGS.FULL) {
  188. effectiveScaleFactors[1] = 3.0; // 360p
  189. effectiveScaleFactors[0] = 6.0; // 180p
  190. }
  191. }
  192. const maxBitrate = videoType === VideoType.DESKTOP
  193. ? codecBitrates.ssHigh : cameraMaxbitrate;
  194. let effectiveBitrates = [ codecBitrates.low, codecBitrates.standard, maxBitrate ];
  195. // The SSRCs on older versions of Firefox are reversed in SDP, i.e., they have resolution order of 1:2:4 as
  196. // opposed to Chromium and other browsers. This has been reverted in Firefox 117 as part of the below commit.
  197. // https://hg.mozilla.org/mozilla-central/rev/b0348f1f8d7197fb87158ba74542d28d46133997
  198. // This revert seems to be applied only to camera tracks, the desktop stream encodings still have the
  199. // resolution order of 4:2:1.
  200. if (browser.isFirefox()
  201. && !browser.supportsScalabilityModeAPI()
  202. && (videoType === VideoType.DESKTOP || browser.isVersionLessThan(117))) {
  203. effectiveBitrates = effectiveBitrates.reverse();
  204. effectiveScaleFactors = effectiveScaleFactors.reverse();
  205. }
  206. const standardSimulcastEncodings = [
  207. {
  208. active: this.pc.videoTransferActive,
  209. maxBitrate: effectiveBitrates[0],
  210. rid: SIM_LAYERS[0].rid,
  211. scaleResolutionDownBy: effectiveScaleFactors[0]
  212. },
  213. {
  214. active: this.pc.videoTransferActive,
  215. maxBitrate: effectiveBitrates[1],
  216. rid: SIM_LAYERS[1].rid,
  217. scaleResolutionDownBy: effectiveScaleFactors[1]
  218. },
  219. {
  220. active: this.pc.videoTransferActive,
  221. maxBitrate: effectiveBitrates[2],
  222. rid: SIM_LAYERS[2].rid,
  223. scaleResolutionDownBy: effectiveScaleFactors[2]
  224. }
  225. ];
  226. if (this.codecSettings[codec].scalabilityModeEnabled) {
  227. // Configure all 3 encodings when simulcast is requested through config.js for AV1 and VP9 and for H.264
  228. // always since that is the only supported mode when DD header extension is negotiated for H.264.
  229. if (this.codecSettings[codec].useSimulcast || codec === CodecMimeType.H264) {
  230. for (const encoding of standardSimulcastEncodings) {
  231. encoding.scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  232. }
  233. return standardSimulcastEncodings;
  234. }
  235. // Configure only one encoding for the SVC mode.
  236. return [
  237. {
  238. active: this.pc.videoTransferActive,
  239. maxBitrate: effectiveBitrates[2],
  240. rid: SIM_LAYERS[0].rid,
  241. scaleResolutionDownBy: effectiveScaleFactors[2],
  242. scalabilityMode: this.codecSettings[codec].useKSVC
  243. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3
  244. },
  245. {
  246. active: false,
  247. maxBitrate: 0
  248. },
  249. {
  250. active: false,
  251. maxBitrate: 0
  252. }
  253. ];
  254. }
  255. return standardSimulcastEncodings;
  256. }
  257. /**
  258. * Returns a boolean indicating whether the video encoder is running in full SVC mode, i.e., it sends only one
  259. * video stream that has both temporal and spatial scalability.
  260. *
  261. * @param {CodecMimeType} codec - The video codec in use.
  262. * @returns boolean - true if the video encoder is running in full SVC mode, false otherwise.
  263. * @private
  264. */
  265. _isRunningInFullSvcMode(codec) {
  266. return (codec === CodecMimeType.VP9 || codec === CodecMimeType.AV1)
  267. && this.codecSettings[codec].scalabilityModeEnabled
  268. && !this.codecSettings[codec].useSimulcast;
  269. }
  270. /**
  271. * Returns a boolean indicating whether the bitrate needs to be capped for the local video track if it happens to
  272. * be a screenshare track. The lower spatial layers for screensharing are disabled when low fps screensharing is in
  273. * progress. Sending all three streams often results in the browser suspending the high resolution in low b/w and
  274. * and low cpu conditions, especially on the low end machines. Suspending the low resolution streams ensures that
  275. * the highest resolution stream is available always. Safari is an exception here since it does not send the
  276. * desktop stream at all if only the high resolution stream is enabled.
  277. *
  278. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  279. * @returns {boolean} - true if the bitrate needs to be capped for the screenshare track, false otherwise.
  280. * @private
  281. */
  282. _isScreenshareBitrateCapped(localVideoTrack) {
  283. return localVideoTrack.getVideoType() === VideoType.DESKTOP
  284. && this.pc._capScreenshareBitrate
  285. && !browser.isWebKitBased();
  286. }
  287. /**
  288. * Returns the calculated active state of the stream encodings based on the frame height requested for the send
  289. * stream. All the encodings that have a resolution lower than the frame height requested will be enabled.
  290. *
  291. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  292. * @param {CodecMimeType} codec - The codec currently in use.
  293. * @param {number} newHeight The resolution requested for the video track.
  294. * @returns {Array<boolean>}
  295. */
  296. calculateEncodingsActiveState(localVideoTrack, codec, newHeight) {
  297. const height = localVideoTrack.getCaptureResolution();
  298. const videoStreamEncodings = this._getVideoStreamEncodings(localVideoTrack, codec);
  299. const encodingsState = videoStreamEncodings
  300. .map(encoding => height / encoding.scaleResolutionDownBy)
  301. .map((frameHeight, idx) => {
  302. let activeState = false;
  303. // When video is suspended on the media session.
  304. if (!this.pc.videoTransferActive) {
  305. return activeState;
  306. }
  307. // Single video stream.
  308. if (!this.pc.isSpatialScalabilityOn() || this._isRunningInFullSvcMode(codec)) {
  309. const { active } = this._calculateActiveEncodingParams(localVideoTrack, codec, newHeight);
  310. return idx === 0 ? active : activeState;
  311. }
  312. if (newHeight > 0) {
  313. if (localVideoTrack.getVideoType() === VideoType.CAMERA) {
  314. activeState = frameHeight <= newHeight
  315. // Keep the LD stream enabled even when the LD stream's resolution is higher than of the
  316. // requested resolution. This can happen when camera is captured at high resolutions like 4k
  317. // but the requested resolution is 180. Since getParameters doesn't give us information about
  318. // the resolutions of the simulcast encodings, we have to rely on our initial config for the
  319. // simulcast streams.
  320. || videoStreamEncodings[idx]?.scaleResolutionDownBy === SIM_LAYERS[0].scaleFactor;
  321. } else {
  322. // For screenshare, keep the HD layer enabled always and the lower layers only for high fps
  323. // screensharing.
  324. activeState = videoStreamEncodings[idx].scaleResolutionDownBy === SIM_LAYERS[2].scaleFactor
  325. || !this._isScreenshareBitrateCapped(localVideoTrack);
  326. }
  327. }
  328. return activeState;
  329. });
  330. return encodingsState;
  331. }
  332. /**
  333. * Returns the calculated max bitrates that need to be configured on the stream encodings based on the video
  334. * type and other considerations associated with screenshare.
  335. *
  336. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  337. * @param {CodecMimeType} codec - The codec currently in use.
  338. * @param {number} newHeight The resolution requested for the video track.
  339. * @returns {Array<number>}
  340. */
  341. calculateEncodingsBitrates(localVideoTrack, codec, newHeight) {
  342. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  343. const desktopShareBitrate = this.options.videoQuality?.desktopbitrate || codecBitrates.ssHigh;
  344. const encodingsBitrates = this._getVideoStreamEncodings(localVideoTrack, codec)
  345. .map((encoding, idx) => {
  346. let bitrate = encoding.maxBitrate;
  347. // Single video stream.
  348. if (!this.pc.isSpatialScalabilityOn() || this._isRunningInFullSvcMode(codec)) {
  349. const { maxBitrate } = this._calculateActiveEncodingParams(localVideoTrack, codec, newHeight);
  350. return idx === 0 ? maxBitrate : 0;
  351. }
  352. // Multiple video streams.
  353. if (this._isScreenshareBitrateCapped(localVideoTrack)) {
  354. bitrate = desktopShareBitrate;
  355. }
  356. return bitrate;
  357. });
  358. return encodingsBitrates;
  359. }
  360. /**
  361. * Returns the calculated scalability modes for the video encodings when scalability modes are supported.
  362. *
  363. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  364. * @param {CodecMimeType} codec - The codec currently in use.
  365. * @param {number} maxHeight The resolution requested for the video track.
  366. * @returns {Array<VideoEncoderScalabilityMode> | undefined}
  367. */
  368. calculateEncodingsScalabilityMode(localVideoTrack, codec, maxHeight) {
  369. if (!this.pc.isSpatialScalabilityOn() || !this.codecSettings[codec].scalabilityModeEnabled) {
  370. return;
  371. }
  372. // Default modes for simulcast.
  373. const scalabilityModes = [
  374. VideoEncoderScalabilityMode.L1T3,
  375. VideoEncoderScalabilityMode.L1T3,
  376. VideoEncoderScalabilityMode.L1T3
  377. ];
  378. // Full SVC mode.
  379. if (this._isRunningInFullSvcMode(codec)) {
  380. const { scalabilityMode }
  381. = this._calculateActiveEncodingParams(localVideoTrack, codec, maxHeight);
  382. scalabilityModes[0] = scalabilityMode;
  383. scalabilityModes[1] = undefined;
  384. scalabilityModes[2] = undefined;
  385. return scalabilityModes;
  386. }
  387. return scalabilityModes;
  388. }
  389. /**
  390. * Returns the scale factor that needs to be applied on the local video stream based on the desired resolution
  391. * and the codec in use.
  392. *
  393. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  394. * @param {CodecMimeType} codec - The codec currently in use.
  395. * @param {number} maxHeight The resolution requested for the video track.
  396. * @returns {Array<float>}
  397. */
  398. calculateEncodingsScaleFactor(localVideoTrack, codec, maxHeight) {
  399. if (this.pc.isSpatialScalabilityOn() && this.isRunningInSimulcastMode(codec)) {
  400. return this._getVideoStreamEncodings(localVideoTrack, codec)
  401. .map(encoding => encoding.scaleResolutionDownBy);
  402. }
  403. // Single video stream.
  404. const { scaleResolutionDownBy }
  405. = this._calculateActiveEncodingParams(localVideoTrack, codec, maxHeight);
  406. return [ scaleResolutionDownBy, undefined, undefined ];
  407. }
  408. /**
  409. * Ensures that the ssrcs associated with a FID ssrc-group appear in the correct order, i.e., the primary ssrc
  410. * first and the secondary rtx ssrc later. This is important for unified plan since we have only one FID group per
  411. * media description.
  412. * @param {Object} description the webRTC session description instance for the remote description.
  413. * @returns {Object} the modified webRTC session description instance.
  414. */
  415. ensureCorrectOrderOfSsrcs(description) {
  416. const parsedSdp = transform.parse(description.sdp);
  417. parsedSdp.media.forEach(mLine => {
  418. if (mLine.type === MediaType.AUDIO) {
  419. return;
  420. }
  421. if (!mLine.ssrcGroups || !mLine.ssrcGroups.length) {
  422. return;
  423. }
  424. let reorderedSsrcs = [];
  425. const ssrcs = new Set();
  426. mLine.ssrcGroups.map(group =>
  427. group.ssrcs
  428. .split(' ')
  429. .filter(Boolean)
  430. .forEach(ssrc => ssrcs.add(ssrc))
  431. );
  432. ssrcs.forEach(ssrc => {
  433. const sources = mLine.ssrcs.filter(source => source.id.toString() === ssrc);
  434. reorderedSsrcs = reorderedSsrcs.concat(sources);
  435. });
  436. mLine.ssrcs = reorderedSsrcs;
  437. });
  438. return {
  439. type: description.type,
  440. sdp: transform.write(parsedSdp)
  441. };
  442. }
  443. /**
  444. * Returns the codec that is configured on the client as the preferred video codec for the given local video track.
  445. *
  446. * @param {JitsiLocalTrack} localTrack - The local video track.
  447. * @returns {CodecMimeType} The codec that is set as the preferred codec for the given local video track.
  448. */
  449. getConfiguredVideoCodec(localTrack) {
  450. const localVideoTrack = localTrack ?? this.pc.getLocalVideoTracks()[0];
  451. const rtpSender = this.pc.findSenderForTrack(localVideoTrack.getTrack());
  452. if (this.pc.usesCodecSelectionAPI() && rtpSender) {
  453. const { encodings } = rtpSender.getParameters();
  454. if (encodings[0].codec) {
  455. return encodings[0].codec.mimeType.split('/')[1].toLowerCase();
  456. }
  457. }
  458. const sdp = this.pc.remoteDescription?.sdp;
  459. if (!sdp) {
  460. return CodecMimeType.VP8;
  461. }
  462. const parsedSdp = transform.parse(sdp);
  463. const mLine = parsedSdp.media
  464. .find(m => m.mid.toString() === this.pc.localTrackTransceiverMids.get(localVideoTrack.rtcId));
  465. const payload = mLine.payloads.split(' ')[0];
  466. const { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload));
  467. if (codec) {
  468. return Object.values(CodecMimeType).find(value => value === codec.toLowerCase());
  469. }
  470. return CodecMimeType.VP8;
  471. }
  472. /**
  473. * Returns the codecs in the current order of preference as configured on the peerconnection.
  474. *
  475. * @param {string} - The local SDP to be used.
  476. * @returns {Array}
  477. */
  478. getConfiguredVideoCodecs(sdp) {
  479. const currentSdp = sdp ?? this.pc.localDescription?.sdp;
  480. if (!currentSdp) {
  481. return [];
  482. }
  483. const parsedSdp = transform.parse(currentSdp);
  484. return this._getConfiguredVideoCodecsImpl(parsedSdp);
  485. }
  486. /**
  487. * Returns the desired media direction for the given media type based on the current state of the peerconnection.
  488. *
  489. * @param {MediaType} mediaType - The media type for which the desired media direction is to be obtained.
  490. * @param {boolean} isAddOperation - Whether the direction is being set for a source add operation.
  491. * @returns {MediaDirection} - The desired media direction for the given media type.
  492. */
  493. getDesiredMediaDirection(mediaType, isAddOperation = false) {
  494. const hasLocalSource = this.pc.getLocalTracks(mediaType).length > 0;
  495. if (isAddOperation) {
  496. return hasLocalSource ? MediaDirection.SENDRECV : MediaDirection.SENDONLY;
  497. }
  498. return hasLocalSource ? MediaDirection.RECVONLY : MediaDirection.INACTIVE;
  499. }
  500. /**
  501. * Obtains stream encodings that need to be configured on the given track based
  502. * on the track media type and the simulcast setting.
  503. * @param {JitsiLocalTrack} localTrack
  504. */
  505. getStreamEncodings(localTrack) {
  506. if (localTrack.isAudioTrack()) {
  507. return [ { active: this.pc.audioTransferActive } ];
  508. }
  509. const codec = this.getConfiguredVideoCodec(localTrack);
  510. if (this.pc.isSpatialScalabilityOn()) {
  511. return this._getVideoStreamEncodings(localTrack, codec);
  512. }
  513. return [ {
  514. active: this.pc.videoTransferActive,
  515. maxBitrate: this.codecSettings[codec].maxBitratesVideo.high
  516. } ];
  517. }
  518. /**
  519. * Injects a 'SIM' ssrc-group line for simulcast into the given session description object to make Jicofo happy.
  520. * This is needed only for Firefox since it does not generate it when simulcast is enabled but we run the check
  521. * on all browsers just in case as it would break the functionality otherwise.
  522. *
  523. * @param desc A session description object (with 'type' and 'sdp' fields)
  524. * @return A session description object with its sdp field modified to contain an inject ssrc-group for simulcast.
  525. */
  526. injectSsrcGroupForSimulcast(desc) {
  527. const sdp = transform.parse(desc.sdp);
  528. const video = sdp.media.find(mline => mline.type === 'video');
  529. // Check if the browser supports RTX, add only the primary ssrcs to the SIM group if that is the case.
  530. video.ssrcGroups = video.ssrcGroups || [];
  531. const fidGroups = video.ssrcGroups.filter(group => group.semantics === SSRC_GROUP_SEMANTICS.FID);
  532. if (video.simulcast || video.simulcast_03) {
  533. const ssrcs = [];
  534. if (fidGroups && fidGroups.length) {
  535. fidGroups.forEach(group => {
  536. ssrcs.push(group.ssrcs.split(' ')[0]);
  537. });
  538. } else {
  539. video.ssrcs.forEach(ssrc => {
  540. if (ssrc.attribute === 'msid') {
  541. ssrcs.push(ssrc.id);
  542. }
  543. });
  544. }
  545. if (video.ssrcGroups.find(group => group.semantics === SSRC_GROUP_SEMANTICS.SIM)) {
  546. // Group already exists, no need to do anything
  547. return desc;
  548. }
  549. // Add a SIM group for every 3 FID groups.
  550. for (let i = 0; i < ssrcs.length; i += 3) {
  551. const simSsrcs = ssrcs.slice(i, i + 3);
  552. video.ssrcGroups.push({
  553. semantics: SSRC_GROUP_SEMANTICS.SIM,
  554. ssrcs: simSsrcs.join(' ')
  555. });
  556. }
  557. }
  558. return {
  559. type: desc.type,
  560. sdp: transform.write(sdp)
  561. };
  562. }
  563. /**
  564. * Takes in a *unified plan* offer and inserts the appropriate parameters for adding simulcast receive support.
  565. * @param {Object} desc - A session description object
  566. * @param {String} desc.type - the type (offer/answer)
  567. * @param {String} desc.sdp - the sdp content
  568. *
  569. * @return {Object} A session description (same format as above) object with its sdp field modified to advertise
  570. * simulcast receive support.
  571. */
  572. insertUnifiedPlanSimulcastReceive(desc) {
  573. // a=simulcast line is not needed on browsers where we SDP munging is used for enabling on simulcast.
  574. // Remove this check when the client switches to RID/MID based simulcast on all browsers.
  575. if (browser.usesSdpMungingForSimulcast()) {
  576. return desc;
  577. }
  578. const rids = [
  579. {
  580. id: SIM_LAYERS[0].rid,
  581. direction: 'recv'
  582. },
  583. {
  584. id: SIM_LAYERS[1].rid,
  585. direction: 'recv'
  586. },
  587. {
  588. id: SIM_LAYERS[2].rid,
  589. direction: 'recv'
  590. }
  591. ];
  592. const ridLine = rids.map(val => val.id).join(';');
  593. const simulcastLine = `recv ${ridLine}`;
  594. const sdp = transform.parse(desc.sdp);
  595. const mLines = sdp.media.filter(m => m.type === MediaType.VIDEO);
  596. const senderMids = Array.from(this.pc.localTrackTransceiverMids.values());
  597. mLines.forEach((mLine, idx) => {
  598. // Make sure the simulcast recv line is only set on video descriptions that are associated with senders.
  599. if (senderMids.find(sender => mLine.mid.toString() === sender.toString()) || idx === 0) {
  600. if (!mLine.simulcast_03 || !mLine.simulcast) {
  601. mLine.rids = rids;
  602. // eslint-disable-next-line camelcase
  603. mLine.simulcast_03 = {
  604. value: simulcastLine
  605. };
  606. }
  607. } else {
  608. mLine.rids = undefined;
  609. mLine.simulcast = undefined;
  610. // eslint-disable-next-line camelcase
  611. mLine.simulcast_03 = undefined;
  612. }
  613. });
  614. return {
  615. type: desc.type,
  616. sdp: transform.write(sdp)
  617. };
  618. }
  619. /**
  620. * Returns a boolean indicating whether the video encoder is running in Simulcast mode, i.e., three encodings need
  621. * to be configured in 4:2:1 resolution order with temporal scalability.
  622. *
  623. * @param {CodecMimeType} videoCodec - The video codec in use.
  624. * @returns {boolean}
  625. */
  626. isRunningInSimulcastMode(videoCodec) {
  627. if (!this.codecSettings || !this.codecSettings[videoCodec]) {
  628. // If codec settings are not available, assume no simulcast
  629. return false;
  630. }
  631. return videoCodec === CodecMimeType.VP8 // VP8 always
  632. // K-SVC mode for VP9 when no scalability mode is set. Though only one outbound-rtp stream is present,
  633. // three separate encodings have to be configured.
  634. || (!this.codecSettings[videoCodec].scalabilityModeEnabled && videoCodec === CodecMimeType.VP9)
  635. // When scalability is enabled, always for H.264, and only when simulcast is explicitly enabled via
  636. // config.js for VP9 and AV1 since full SVC is the default mode for these 2 codecs.
  637. || (this.codecSettings[videoCodec].scalabilityModeEnabled
  638. && (videoCodec === CodecMimeType.H264 || this.codecSettings[videoCodec].useSimulcast));
  639. }
  640. /**
  641. * Munges the session description to ensure that the codec order is as per the preferred codec settings.
  642. *
  643. * @param {transform.SessionDescription} parsedSdp that needs to be munged
  644. * @returns {transform.SessionDescription} the munged SDP.
  645. */
  646. mungeCodecOrder(parsedSdp) {
  647. const codecSettings = this.pc.codecSettings;
  648. if (!codecSettings) {
  649. return parsedSdp;
  650. }
  651. const mungedSdp = parsedSdp;
  652. const { isP2P } = this.options;
  653. const mLines = mungedSdp.media.filter(m => m.type === codecSettings.mediaType);
  654. for (const mLine of mLines) {
  655. const currentCodecs = this._getConfiguredVideoCodecsImpl(mungedSdp);
  656. for (const codec of currentCodecs) {
  657. if (isP2P) {
  658. // 1. Strip the high profile H264 codecs on all clients. macOS started offering encoder for H.264
  659. // level 5.2 but a decoder only for level 3.1. Therfore, strip all main and high level codecs for
  660. // H.264.
  661. // 2. There are multiple VP9 payload types generated by the browser, more payload types are added
  662. // if the endpoint doesn't have a local video source. Therefore, strip all the high profile codec
  663. // variants for VP9 so that only one payload type for VP9 is negotiated between the peers.
  664. if (codec === CodecMimeType.H264 || codec === CodecMimeType.VP9) {
  665. SDPUtil.stripCodec(mLine, codec, true /* high profile */);
  666. }
  667. // Do not negotiate ULPFEC and RED either.
  668. if (codec === CodecMimeType.ULPFEC || codec === CodecMimeType.RED) {
  669. SDPUtil.stripCodec(mLine, codec, false);
  670. }
  671. }
  672. }
  673. // Reorder the codecs based on the preferred settings.
  674. if (!this.pc.usesCodecSelectionAPI()) {
  675. for (const codec of codecSettings.codecList.slice().reverse()) {
  676. SDPUtil.preferCodec(mLine, codec, isP2P);
  677. }
  678. }
  679. }
  680. return mungedSdp;
  681. }
  682. /**
  683. * Munges the stereo flag as well as the opusMaxAverageBitrate in the SDP, based on values set through config.js,
  684. * if present.
  685. *
  686. * @param {transform.SessionDescription} parsedSdp that needs to be munged.
  687. * @returns {transform.SessionDescription} the munged SDP.
  688. */
  689. mungeOpus(parsedSdp) {
  690. const { audioQuality } = this.options;
  691. if (!audioQuality?.enableOpusDtx && !audioQuality?.stereo && !audioQuality?.opusMaxAverageBitrate) {
  692. return parsedSdp;
  693. }
  694. const mungedSdp = parsedSdp;
  695. const mLines = mungedSdp.media.filter(m => m.type === MediaType.AUDIO);
  696. for (const mLine of mLines) {
  697. const { payload } = mLine.rtp.find(protocol => protocol.codec === CodecMimeType.OPUS);
  698. if (!payload) {
  699. // eslint-disable-next-line no-continue
  700. continue;
  701. }
  702. let fmtpOpus = mLine.fmtp.find(protocol => protocol.payload === payload);
  703. if (!fmtpOpus) {
  704. fmtpOpus = {
  705. payload,
  706. config: ''
  707. };
  708. }
  709. const fmtpConfig = transform.parseParams(fmtpOpus.config);
  710. let sdpChanged = false;
  711. if (audioQuality?.stereo) {
  712. fmtpConfig.stereo = 1;
  713. sdpChanged = true;
  714. }
  715. if (audioQuality?.opusMaxAverageBitrate) {
  716. fmtpConfig.maxaveragebitrate = audioQuality.opusMaxAverageBitrate;
  717. sdpChanged = true;
  718. }
  719. // On Firefox, the OpusDtx enablement has no effect
  720. if (!browser.isFirefox() && audioQuality?.enableOpusDtx) {
  721. fmtpConfig.usedtx = 1;
  722. sdpChanged = true;
  723. }
  724. if (!sdpChanged) {
  725. // eslint-disable-next-line no-continue
  726. continue;
  727. }
  728. let mungedConfig = '';
  729. for (const key of Object.keys(fmtpConfig)) {
  730. mungedConfig += `${key}=${fmtpConfig[key]}; `;
  731. }
  732. fmtpOpus.config = mungedConfig.trim();
  733. }
  734. return mungedSdp;
  735. }
  736. /**
  737. * Munges the session SDP by setting the max bitrates on the video m-lines when VP9 K-SVC codec is in use.
  738. *
  739. * @param {transform.SessionDescription} parsedSdp that needs to be munged.
  740. * @param {boolean} isLocalSdp - Whether the max bitrate (via b=AS line in SDP) is set on local SDP.
  741. * @returns {transform.SessionDescription} The munged SDP.
  742. */
  743. setMaxBitrates(parsedSdp, isLocalSdp = false) {
  744. const pcCodecSettings = this.pc.codecSettings;
  745. if (!pcCodecSettings) {
  746. return parsedSdp;
  747. }
  748. // Find all the m-lines associated with the local sources.
  749. const mungedSdp = parsedSdp;
  750. const direction = isLocalSdp ? MediaDirection.RECVONLY : MediaDirection.SENDONLY;
  751. const mLines = mungedSdp.media.filter(m => m.type === MediaType.VIDEO && m.direction !== direction);
  752. const currentCodec = pcCodecSettings.codecList[0];
  753. const codecScalabilityModeSettings = this.codecSettings[currentCodec];
  754. for (const mLine of mLines) {
  755. const isDoingVp9KSvc = currentCodec === CodecMimeType.VP9
  756. && !codecScalabilityModeSettings.scalabilityModeEnabled;
  757. const localTrack = this.pc.getLocalVideoTracks()
  758. .find(track => this.pc.localTrackTransceiverMids.get(track.rtcId) === mLine.mid.toString());
  759. if (localTrack
  760. && (isDoingVp9KSvc
  761. // Setting bitrates in the SDP for SVC codecs is no longer needed in the newer versions where
  762. // maxBitrates from the RTCRtpEncodingParameters directly affect the target bitrate for the encoder.
  763. || (this._isRunningInFullSvcMode(currentCodec) && !this.pc.usesCodecSelectionAPI()))) {
  764. let maxBitrate;
  765. if (localTrack.getVideoType() === VideoType.DESKTOP) {
  766. maxBitrate = codecScalabilityModeSettings.maxBitratesVideo.ssHigh;
  767. } else {
  768. const { level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= localTrack.getCaptureResolution());
  769. maxBitrate = codecScalabilityModeSettings.maxBitratesVideo[level];
  770. }
  771. const limit = Math.floor(maxBitrate / 1000);
  772. // Use only the highest spatial layer bitrates for now as there is no API available yet for configuring
  773. // the bitrates on the individual SVC layers.
  774. mLine.bandwidth = [ {
  775. type: 'AS',
  776. limit
  777. } ];
  778. } else {
  779. // Clear the bandwidth limit in SDP when VP9 is no longer the preferred codec.
  780. // This is needed on react native clients as react-native-webrtc returns the
  781. // SDP that the application passed instead of returning the SDP off the native side.
  782. // This line automatically gets cleared on web on every renegotiation.
  783. mLine.bandwidth = undefined;
  784. }
  785. }
  786. return mungedSdp;
  787. }
  788. /**
  789. * Checks if the AV1 Dependency descriptors are negotiated on the bridge peerconnection and removes them from the
  790. * SDP when codec selected is VP8 or VP9.
  791. *
  792. * @param {transform.SessionDescription} parsedSdp that needs to be munged.
  793. * @returns {string} the munged SDP.
  794. */
  795. updateAv1DdHeaders(parsedSdp) {
  796. if (!browser.supportsDDExtHeaders()) {
  797. return parsedSdp;
  798. }
  799. const mungedSdp = parsedSdp;
  800. const mLines = mungedSdp.media.filter(m => m.type === MediaType.VIDEO);
  801. mLines.forEach((mLine, idx) => {
  802. const senderMids = Array.from(this.pc.localTrackTransceiverMids.values());
  803. const isSender = senderMids.length
  804. ? senderMids.find(mid => mLine.mid.toString() === mid.toString())
  805. : idx === 0;
  806. const payload = mLine.payloads.split(' ')[0];
  807. let { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload));
  808. codec = codec.toLowerCase();
  809. if (isSender && mLine.ext?.length) {
  810. const headerIndex = mLine.ext.findIndex(ext => ext.uri === DD_HEADER_EXT_URI);
  811. const shouldNegotiateHeaderExts = codec === CodecMimeType.AV1 || codec === CodecMimeType.H264;
  812. if (!this.supportsDDHeaderExt && headerIndex >= 0) {
  813. this.supportsDDHeaderExt = true;
  814. }
  815. if (this.supportsDDHeaderExt && shouldNegotiateHeaderExts && headerIndex < 0) {
  816. mLine.ext.push({
  817. value: DD_HEADER_EXT_ID,
  818. uri: DD_HEADER_EXT_URI
  819. });
  820. } else if (!shouldNegotiateHeaderExts && headerIndex >= 0) {
  821. mLine.ext.splice(headerIndex, 1);
  822. }
  823. }
  824. });
  825. return mungedSdp;
  826. }
  827. }