您最多选择25个主题 主题必须以字母或数字开头,可以包含连字符 (-),并且长度不得超过35个字符

TPCUtils.js 37KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900
  1. import { getLogger } from '@jitsi/logger';
  2. import { cloneDeep } from 'lodash-es';
  3. import transform from 'sdp-transform';
  4. import { CodecMimeType } from '../../service/RTC/CodecMimeType';
  5. import { MediaDirection } from '../../service/RTC/MediaDirection';
  6. import { MediaType } from '../../service/RTC/MediaType';
  7. import {
  8. SIM_LAYERS,
  9. STANDARD_CODEC_SETTINGS,
  10. VIDEO_QUALITY_LEVELS,
  11. VIDEO_QUALITY_SETTINGS
  12. } from '../../service/RTC/StandardVideoQualitySettings';
  13. import { VideoEncoderScalabilityMode } from '../../service/RTC/VideoEncoderScalabilityMode';
  14. import { VideoType } from '../../service/RTC/VideoType';
  15. import browser from '../browser';
  16. import SDPUtil from '../sdp/SDPUtil';
  17. const logger = getLogger(__filename);
  18. const DD_HEADER_EXT_URI
  19. = 'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension';
  20. const DD_HEADER_EXT_ID = 11;
  21. const VIDEO_CODECS = [ CodecMimeType.AV1, CodecMimeType.H264, CodecMimeType.VP8, CodecMimeType.VP9 ];
  22. /**
  23. * Handles all the utility functions for the TraceablePeerConnection class, like calculating the encoding parameters,
  24. * determining the media direction, calculating bitrates based on the current codec settings, etc.
  25. */
  26. export class TPCUtils {
  27. /**
  28. * Creates a new instance for a given TraceablePeerConnection
  29. *
  30. * @param peerconnection - the tpc instance for which we have utility functions.
  31. */
  32. constructor(peerconnection, options) {
  33. this.pc = peerconnection;
  34. this.options = options;
  35. this.codecSettings = cloneDeep(STANDARD_CODEC_SETTINGS);
  36. /**
  37. * Flag indicating bridge support for AV1 codec. On the bridge connection, it is supported only when support for
  38. * Dependency Descriptor header extensions is offered by Jicofo. H.264 simulcast is also possible when these
  39. * header extensions are negotiated.
  40. */
  41. this.supportsDDHeaderExt = false;
  42. /**
  43. * Reads videoQuality settings from config.js and overrides the code defaults for video codecs.
  44. */
  45. const videoQualitySettings = this.options.videoQuality;
  46. if (videoQualitySettings) {
  47. for (const codec of VIDEO_CODECS) {
  48. const codecConfig = videoQualitySettings[codec];
  49. const bitrateSettings = codecConfig?.maxBitratesVideo
  50. // Read the deprecated settings for max bitrates.
  51. ?? (videoQualitySettings.maxbitratesvideo
  52. && videoQualitySettings.maxbitratesvideo[codec.toUpperCase()]);
  53. if (bitrateSettings) {
  54. const settings = Object.values(VIDEO_QUALITY_SETTINGS);
  55. [ ...settings, 'ssHigh' ].forEach(value => {
  56. if (bitrateSettings[value]) {
  57. this.codecSettings[codec].maxBitratesVideo[value] = bitrateSettings[value];
  58. }
  59. });
  60. }
  61. if (!codecConfig) {
  62. continue; // eslint-disable-line no-continue
  63. }
  64. const scalabilityModeEnabled = this.codecSettings[codec].scalabilityModeEnabled
  65. && (typeof codecConfig.scalabilityModeEnabled === 'undefined'
  66. || codecConfig.scalabilityModeEnabled);
  67. if (scalabilityModeEnabled) {
  68. typeof codecConfig.useSimulcast !== 'undefined'
  69. && (this.codecSettings[codec].useSimulcast = codecConfig.useSimulcast);
  70. typeof codecConfig.useKSVC !== 'undefined'
  71. && (this.codecSettings[codec].useKSVC = codecConfig.useKSVC);
  72. } else {
  73. this.codecSettings[codec].scalabilityModeEnabled = false;
  74. }
  75. }
  76. }
  77. }
  78. /**
  79. * Calculates the configuration of the active encoding when the browser sends only one stream, i,e,, when there is
  80. * no spatial scalability configure (p2p) or when it is running in full SVC mode.
  81. *
  82. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  83. * @param {CodecMimeType} codec - The video codec.
  84. * @param {number} newHeight - The resolution that needs to be configured for the local video track.
  85. * @returns {Object} configuration.
  86. */
  87. _calculateActiveEncodingParams(localVideoTrack, codec, newHeight) {
  88. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  89. const trackCaptureHeight = localVideoTrack.getCaptureResolution();
  90. const effectiveNewHeight = newHeight > trackCaptureHeight ? trackCaptureHeight : newHeight;
  91. const desktopShareBitrate = this.options.videoQuality?.desktopbitrate || codecBitrates.ssHigh;
  92. const isScreenshare = localVideoTrack.getVideoType() === VideoType.DESKTOP;
  93. let scalabilityMode = this.codecSettings[codec].useKSVC
  94. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3;
  95. const { height, level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= effectiveNewHeight);
  96. let maxBitrate;
  97. let scaleResolutionDownBy = SIM_LAYERS[2].scaleFactor;
  98. if (this._isScreenshareBitrateCapped(localVideoTrack)) {
  99. scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  100. maxBitrate = desktopShareBitrate;
  101. } else if (isScreenshare) {
  102. maxBitrate = codecBitrates.ssHigh;
  103. } else {
  104. maxBitrate = codecBitrates[level];
  105. effectiveNewHeight && (scaleResolutionDownBy = trackCaptureHeight / effectiveNewHeight);
  106. if (height !== effectiveNewHeight) {
  107. logger.debug(`Quality level with height=${height} was picked when requested height=${newHeight} for`
  108. + `track with capture height=${trackCaptureHeight}`);
  109. }
  110. }
  111. const config = {
  112. active: effectiveNewHeight > 0,
  113. maxBitrate,
  114. scalabilityMode,
  115. scaleResolutionDownBy
  116. };
  117. if (!config.active || isScreenshare) {
  118. return config;
  119. }
  120. // Configure the sender to send all 3 spatial layers for resolutions 720p and higher.
  121. switch (level) {
  122. case VIDEO_QUALITY_SETTINGS.ULTRA:
  123. case VIDEO_QUALITY_SETTINGS.FULL:
  124. case VIDEO_QUALITY_SETTINGS.HIGH:
  125. config.scalabilityMode = this.codecSettings[codec].useKSVC
  126. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3;
  127. break;
  128. case VIDEO_QUALITY_SETTINGS.STANDARD:
  129. config.scalabilityMode = this.codecSettings[codec].useKSVC
  130. ? VideoEncoderScalabilityMode.L2T3_KEY : VideoEncoderScalabilityMode.L2T3;
  131. break;
  132. default:
  133. config.scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  134. }
  135. return config;
  136. }
  137. /**
  138. * The startup configuration for the stream encodings that are applicable to the video stream when a new sender is
  139. * created on the peerconnection. The initial config takes into account the differences in browser's simulcast
  140. * implementation.
  141. *
  142. * Encoding parameters:
  143. * active - determine the on/off state of a particular encoding.
  144. * maxBitrate - max. bitrate value to be applied to that particular encoding based on the encoding's resolution and
  145. * config.js videoQuality settings if applicable.
  146. * rid - Rtp Stream ID that is configured for a particular simulcast stream.
  147. * scaleResolutionDownBy - the factor by which the encoding is scaled down from the original resolution of the
  148. * captured video.
  149. *
  150. * @param {JitsiLocalTrack} localTrack
  151. * @param {String} codec
  152. */
  153. _getVideoStreamEncodings(localTrack, codec) {
  154. const captureResolution = localTrack.getCaptureResolution();
  155. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  156. const videoType = localTrack.getVideoType();
  157. let effectiveScaleFactors = SIM_LAYERS.map(sim => sim.scaleFactor);
  158. let cameraMaxbitrate;
  159. if (videoType === VideoType.CAMERA) {
  160. const { level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= captureResolution);
  161. cameraMaxbitrate = codecBitrates[level];
  162. if (level === VIDEO_QUALITY_SETTINGS.ULTRA) {
  163. effectiveScaleFactors[1] = 6.0; // 360p
  164. effectiveScaleFactors[0] = 12.0; // 180p
  165. } else if (level === VIDEO_QUALITY_SETTINGS.FULL) {
  166. effectiveScaleFactors[1] = 3.0; // 360p
  167. effectiveScaleFactors[0] = 6.0; // 180p
  168. }
  169. }
  170. const maxBitrate = videoType === VideoType.DESKTOP
  171. ? codecBitrates.ssHigh : cameraMaxbitrate;
  172. let effectiveBitrates = [ codecBitrates.low, codecBitrates.standard, maxBitrate ];
  173. // The SSRCs on older versions of Firefox are reversed in SDP, i.e., they have resolution order of 1:2:4 as
  174. // opposed to Chromium and other browsers. This has been reverted in Firefox 117 as part of the below commit.
  175. // https://hg.mozilla.org/mozilla-central/rev/b0348f1f8d7197fb87158ba74542d28d46133997
  176. // This revert seems to be applied only to camera tracks, the desktop stream encodings still have the
  177. // resolution order of 4:2:1.
  178. if (browser.isFirefox() && (videoType === VideoType.DESKTOP || browser.isVersionLessThan(117))) {
  179. effectiveBitrates = effectiveBitrates.reverse();
  180. effectiveScaleFactors = effectiveScaleFactors.reverse();
  181. }
  182. const standardSimulcastEncodings = [
  183. {
  184. active: this.pc.videoTransferActive,
  185. maxBitrate: effectiveBitrates[0],
  186. rid: SIM_LAYERS[0].rid,
  187. scaleResolutionDownBy: effectiveScaleFactors[0]
  188. },
  189. {
  190. active: this.pc.videoTransferActive,
  191. maxBitrate: effectiveBitrates[1],
  192. rid: SIM_LAYERS[1].rid,
  193. scaleResolutionDownBy: effectiveScaleFactors[1]
  194. },
  195. {
  196. active: this.pc.videoTransferActive,
  197. maxBitrate: effectiveBitrates[2],
  198. rid: SIM_LAYERS[2].rid,
  199. scaleResolutionDownBy: effectiveScaleFactors[2]
  200. }
  201. ];
  202. if (this.codecSettings[codec].scalabilityModeEnabled) {
  203. // Configure all 3 encodings when simulcast is requested through config.js for AV1 and VP9 and for H.264
  204. // always since that is the only supported mode when DD header extension is negotiated for H.264.
  205. if (this.codecSettings[codec].useSimulcast || codec === CodecMimeType.H264) {
  206. for (const encoding of standardSimulcastEncodings) {
  207. encoding.scalabilityMode = VideoEncoderScalabilityMode.L1T3;
  208. }
  209. return standardSimulcastEncodings;
  210. }
  211. // Configure only one encoding for the SVC mode.
  212. return [
  213. {
  214. active: this.pc.videoTransferActive,
  215. maxBitrate: effectiveBitrates[2],
  216. rid: SIM_LAYERS[0].rid,
  217. scaleResolutionDownBy: effectiveScaleFactors[2],
  218. scalabilityMode: this.codecSettings[codec].useKSVC
  219. ? VideoEncoderScalabilityMode.L3T3_KEY : VideoEncoderScalabilityMode.L3T3
  220. },
  221. {
  222. active: false,
  223. maxBitrate: 0
  224. },
  225. {
  226. active: false,
  227. maxBitrate: 0
  228. }
  229. ];
  230. }
  231. return standardSimulcastEncodings;
  232. }
  233. /**
  234. * Returns a boolean indicating whether the video encoder is running in full SVC mode, i.e., it sends only one
  235. * video stream that has both temporal and spatial scalability.
  236. *
  237. * @param {CodecMimeType} codec
  238. * @returns boolean
  239. */
  240. _isRunningInFullSvcMode(codec) {
  241. return (codec === CodecMimeType.VP9 || codec === CodecMimeType.AV1)
  242. && this.codecSettings[codec].scalabilityModeEnabled
  243. && !this.codecSettings[codec].useSimulcast;
  244. }
  245. /**
  246. * Returns a boolean indicating whether the bitrate needs to be capped for the local video track if it happens to
  247. * be a screenshare track. The lower spatial layers for screensharing are disabled when low fps screensharing is in
  248. * progress. Sending all three streams often results in the browser suspending the high resolution in low b/w and
  249. * and low cpu conditions, especially on the low end machines. Suspending the low resolution streams ensures that
  250. * the highest resolution stream is available always. Safari is an exception here since it does not send the
  251. * desktop stream at all if only the high resolution stream is enabled.
  252. *
  253. * @param {JitsiLocalTrack} localVideoTrack - The local video track.
  254. * @returns {boolean}
  255. */
  256. _isScreenshareBitrateCapped(localVideoTrack) {
  257. return localVideoTrack.getVideoType() === VideoType.DESKTOP
  258. && this.pc._capScreenshareBitrate
  259. && !browser.isWebKitBased();
  260. }
  261. /**
  262. * Returns the calculated active state of the stream encodings based on the frame height requested for the send
  263. * stream. All the encodings that have a resolution lower than the frame height requested will be enabled.
  264. *
  265. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  266. * @param {CodecMimeType} codec - The codec currently in use.
  267. * @param {number} newHeight The resolution requested for the video track.
  268. * @returns {Array<boolean>}
  269. */
  270. calculateEncodingsActiveState(localVideoTrack, codec, newHeight) {
  271. const height = localVideoTrack.getCaptureResolution();
  272. const videoStreamEncodings = this._getVideoStreamEncodings(localVideoTrack, codec);
  273. const encodingsState = videoStreamEncodings
  274. .map(encoding => height / encoding.scaleResolutionDownBy)
  275. .map((frameHeight, idx) => {
  276. let activeState = false;
  277. // When video is suspended on the media session.
  278. if (!this.pc.videoTransferActive) {
  279. return activeState;
  280. }
  281. // Single video stream.
  282. if (!this.pc.isSpatialScalabilityOn() || this._isRunningInFullSvcMode(codec)) {
  283. const { active } = this._calculateActiveEncodingParams(localVideoTrack, codec, newHeight);
  284. return idx === 0 ? active : activeState;
  285. }
  286. if (newHeight > 0) {
  287. if (localVideoTrack.getVideoType() === VideoType.CAMERA) {
  288. activeState = frameHeight <= newHeight
  289. // Keep the LD stream enabled even when the LD stream's resolution is higher than of the
  290. // requested resolution. This can happen when camera is captured at high resolutions like 4k
  291. // but the requested resolution is 180. Since getParameters doesn't give us information about
  292. // the resolutions of the simulcast encodings, we have to rely on our initial config for the
  293. // simulcast streams.
  294. || videoStreamEncodings[idx]?.scaleResolutionDownBy === SIM_LAYERS[0].scaleFactor;
  295. } else {
  296. // For screenshare, keep the HD layer enabled always and the lower layers only for high fps
  297. // screensharing.
  298. activeState = videoStreamEncodings[idx].scaleResolutionDownBy === SIM_LAYERS[2].scaleFactor
  299. || !this._isScreenshareBitrateCapped(localVideoTrack);
  300. }
  301. }
  302. return activeState;
  303. });
  304. return encodingsState;
  305. }
  306. /**
  307. * Returns the calculated max bitrates that need to be configured on the stream encodings based on the video
  308. * type and other considerations associated with screenshare.
  309. *
  310. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  311. * @param {CodecMimeType} codec - The codec currently in use.
  312. * @param {number} newHeight The resolution requested for the video track.
  313. * @returns {Array<number>}
  314. */
  315. calculateEncodingsBitrates(localVideoTrack, codec, newHeight) {
  316. const codecBitrates = this.codecSettings[codec].maxBitratesVideo;
  317. const desktopShareBitrate = this.options.videoQuality?.desktopbitrate || codecBitrates.ssHigh;
  318. const encodingsBitrates = this._getVideoStreamEncodings(localVideoTrack, codec)
  319. .map((encoding, idx) => {
  320. let bitrate = encoding.maxBitrate;
  321. // Single video stream.
  322. if (!this.pc.isSpatialScalabilityOn() || this._isRunningInFullSvcMode(codec)) {
  323. const { maxBitrate } = this._calculateActiveEncodingParams(localVideoTrack, codec, newHeight);
  324. return idx === 0 ? maxBitrate : 0;
  325. }
  326. // Multiple video streams.
  327. if (this._isScreenshareBitrateCapped(localVideoTrack)) {
  328. bitrate = desktopShareBitrate;
  329. }
  330. return bitrate;
  331. });
  332. return encodingsBitrates;
  333. }
  334. /**
  335. * Returns the calculated scalability modes for the video encodings when scalability modes are supported.
  336. *
  337. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  338. * @param {CodecMimeType} codec - The codec currently in use.
  339. * @param {number} maxHeight The resolution requested for the video track.
  340. * @returns {Array<VideoEncoderScalabilityMode> | undefined}
  341. */
  342. calculateEncodingsScalabilityMode(localVideoTrack, codec, maxHeight) {
  343. if (!this.pc.isSpatialScalabilityOn() || !this.codecSettings[codec].scalabilityModeEnabled) {
  344. return;
  345. }
  346. // Default modes for simulcast.
  347. const scalabilityModes = [
  348. VideoEncoderScalabilityMode.L1T3,
  349. VideoEncoderScalabilityMode.L1T3,
  350. VideoEncoderScalabilityMode.L1T3
  351. ];
  352. // Full SVC mode.
  353. if (this._isRunningInFullSvcMode(codec)) {
  354. const { scalabilityMode }
  355. = this._calculateActiveEncodingParams(localVideoTrack, codec, maxHeight);
  356. scalabilityModes[0] = scalabilityMode;
  357. scalabilityModes[1] = undefined;
  358. scalabilityModes[2] = undefined;
  359. return scalabilityModes;
  360. }
  361. return scalabilityModes;
  362. }
  363. /**
  364. * Returns the scale factor that needs to be applied on the local video stream based on the desired resolution
  365. * and the codec in use.
  366. *
  367. * @param {JitsiLocalTrack} localVideoTrack The local video track.
  368. * @param {CodecMimeType} codec - The codec currently in use.
  369. * @param {number} maxHeight The resolution requested for the video track.
  370. * @returns {Array<float>}
  371. */
  372. calculateEncodingsScaleFactor(localVideoTrack, codec, maxHeight) {
  373. if (this.pc.isSpatialScalabilityOn() && this.isRunningInSimulcastMode(codec)) {
  374. return this._getVideoStreamEncodings(localVideoTrack, codec)
  375. .map(encoding => encoding.scaleResolutionDownBy);
  376. }
  377. // Single video stream.
  378. const { scaleResolutionDownBy }
  379. = this._calculateActiveEncodingParams(localVideoTrack, codec, maxHeight);
  380. return [ scaleResolutionDownBy, undefined, undefined ];
  381. }
  382. /**
  383. * Ensures that the ssrcs associated with a FID ssrc-group appear in the correct order, i.e.,
  384. * the primary ssrc first and the secondary rtx ssrc later. This is important for unified
  385. * plan since we have only one FID group per media description.
  386. * @param {Object} description the webRTC session description instance for the remote
  387. * description.
  388. * @private
  389. */
  390. ensureCorrectOrderOfSsrcs(description) {
  391. const parsedSdp = transform.parse(description.sdp);
  392. parsedSdp.media.forEach(mLine => {
  393. if (mLine.type === MediaType.AUDIO) {
  394. return;
  395. }
  396. if (!mLine.ssrcGroups || !mLine.ssrcGroups.length) {
  397. return;
  398. }
  399. let reorderedSsrcs = [];
  400. const ssrcs = new Set();
  401. mLine.ssrcGroups.map(group =>
  402. group.ssrcs
  403. .split(' ')
  404. .filter(Boolean)
  405. .forEach(ssrc => ssrcs.add(ssrc))
  406. );
  407. ssrcs.forEach(ssrc => {
  408. const sources = mLine.ssrcs.filter(source => source.id.toString() === ssrc);
  409. reorderedSsrcs = reorderedSsrcs.concat(sources);
  410. });
  411. mLine.ssrcs = reorderedSsrcs;
  412. });
  413. return new RTCSessionDescription({
  414. type: description.type,
  415. sdp: transform.write(parsedSdp)
  416. });
  417. }
  418. /**
  419. * Returns the codec that is configured on the client as the preferred video codec for the given local video track.
  420. *
  421. * @param {JitsiLocalTrack} localTrack - The local video track.
  422. * @returns {CodecMimeType} The codec that is set as the preferred codec for the given local video track.
  423. */
  424. getConfiguredVideoCodec(localTrack) {
  425. const localVideoTrack = localTrack ?? this.pc.getLocalVideoTracks()[0];
  426. const rtpSender = this.pc.findSenderForTrack(localVideoTrack.getTrack());
  427. if (this.pc.usesCodecSelectionAPI() && rtpSender) {
  428. const { codecs } = rtpSender.getParameters();
  429. return codecs[0].mimeType.split('/')[1].toLowerCase();
  430. }
  431. const sdp = this.pc.remoteDescription?.sdp;
  432. const defaultCodec = CodecMimeType.VP8;
  433. if (!sdp) {
  434. return defaultCodec;
  435. }
  436. const parsedSdp = transform.parse(sdp);
  437. const mLine = parsedSdp.media
  438. .find(m => m.mid.toString() === this.pc.localTrackTransceiverMids.get(localVideoTrack.rtcId));
  439. const payload = mLine.payloads.split(' ')[0];
  440. const { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload));
  441. if (codec) {
  442. return Object.values(CodecMimeType).find(value => value === codec.toLowerCase());
  443. }
  444. return defaultCodec;
  445. }
  446. /**
  447. * Returns the codecs in the current order of preference as configured on the peerconnection.
  448. *
  449. * @param {RTCSessionDescription} - The local description to be used.
  450. * @returns {Array}
  451. */
  452. getConfiguredVideoCodecs(description) {
  453. const currentSdp = description?.sdp ?? this.pc.localDescription?.sdp;
  454. if (!currentSdp) {
  455. return [];
  456. }
  457. const parsedSdp = transform.parse(currentSdp);
  458. const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO);
  459. const codecs = new Set(mLine.rtp
  460. .filter(pt => pt.codec.toLowerCase() !== 'rtx')
  461. .map(pt => pt.codec.toLowerCase()));
  462. return Array.from(codecs);
  463. }
  464. /**
  465. * Returns the desired media direction for the given media type based on the current state of the peerconnection.
  466. *
  467. * @param {MediaType} mediaType - The media type for which the desired media direction is to be obtained.
  468. * @param {boolean} isAddOperation - Whether the direction is being set for a source add operation.
  469. * @returns {MediaDirection} - The desired media direction for the given media type.
  470. */
  471. getDesiredMediaDirection(mediaType, isAddOperation = false) {
  472. const hasLocalSource = this.pc.getLocalTracks(mediaType).length > 0;
  473. if (isAddOperation) {
  474. return hasLocalSource ? MediaDirection.SENDRECV : MediaDirection.SENDONLY;
  475. }
  476. return hasLocalSource ? MediaDirection.RECVONLY : MediaDirection.INACTIVE;
  477. }
  478. /**
  479. * Obtains stream encodings that need to be configured on the given track based
  480. * on the track media type and the simulcast setting.
  481. * @param {JitsiLocalTrack} localTrack
  482. */
  483. getStreamEncodings(localTrack) {
  484. if (localTrack.isAudioTrack()) {
  485. return [ { active: this.pc.audioTransferActive } ];
  486. }
  487. const codec = this.getConfiguredVideoCodec(localTrack);
  488. if (this.pc.isSpatialScalabilityOn()) {
  489. return this._getVideoStreamEncodings(localTrack, codec);
  490. }
  491. return [ {
  492. active: this.pc.videoTransferActive,
  493. maxBitrate: this.codecSettings[codec].maxBitratesVideo.high
  494. } ];
  495. }
  496. /**
  497. * Takes in a *unified plan* offer and inserts the appropriate parameters for adding simulcast receive support.
  498. * @param {Object} desc - A session description object
  499. * @param {String} desc.type - the type (offer/answer)
  500. * @param {String} desc.sdp - the sdp content
  501. *
  502. * @return {Object} A session description (same format as above) object with its sdp field modified to advertise
  503. * simulcast receive support.
  504. */
  505. insertUnifiedPlanSimulcastReceive(desc) {
  506. // a=simulcast line is not needed on browsers where we SDP munging is used for enabling on simulcast.
  507. // Remove this check when the client switches to RID/MID based simulcast on all browsers.
  508. if (browser.usesSdpMungingForSimulcast()) {
  509. return desc;
  510. }
  511. const rids = [
  512. {
  513. id: SIM_LAYERS[0].rid,
  514. direction: 'recv'
  515. },
  516. {
  517. id: SIM_LAYERS[1].rid,
  518. direction: 'recv'
  519. },
  520. {
  521. id: SIM_LAYERS[2].rid,
  522. direction: 'recv'
  523. }
  524. ];
  525. const ridLine = rids.map(val => val.id).join(';');
  526. const simulcastLine = `recv ${ridLine}`;
  527. const sdp = transform.parse(desc.sdp);
  528. const mLines = sdp.media.filter(m => m.type === MediaType.VIDEO);
  529. const senderMids = Array.from(this.pc.localTrackTransceiverMids.values());
  530. mLines.forEach((mLine, idx) => {
  531. // Make sure the simulcast recv line is only set on video descriptions that are associated with senders.
  532. if (senderMids.find(sender => mLine.mid.toString() === sender.toString()) || idx === 0) {
  533. if (!mLine.simulcast_03 || !mLine.simulcast) {
  534. mLine.rids = rids;
  535. // eslint-disable-next-line camelcase
  536. mLine.simulcast_03 = {
  537. value: simulcastLine
  538. };
  539. }
  540. } else {
  541. mLine.rids = undefined;
  542. mLine.simulcast = undefined;
  543. // eslint-disable-next-line camelcase
  544. mLine.simulcast_03 = undefined;
  545. }
  546. });
  547. return new RTCSessionDescription({
  548. type: desc.type,
  549. sdp: transform.write(sdp)
  550. });
  551. }
  552. /**
  553. * Returns a boolean indicating whether the video encoder is running in Simulcast mode, i.e., three encodings need
  554. * to be configured in 4:2:1 resolution order with temporal scalability.
  555. *
  556. * @param {CodecMimeType} codec - The video codec in use.
  557. * @returns {boolean}
  558. */
  559. isRunningInSimulcastMode(codec) {
  560. return codec === CodecMimeType.VP8 // VP8 always
  561. // K-SVC mode for VP9 when no scalability mode is set. Though only one outbound-rtp stream is present,
  562. // three separate encodings have to be configured.
  563. || (!this.codecSettings[codec].scalabilityModeEnabled && codec === CodecMimeType.VP9)
  564. // When scalability is enabled, always for H.264, and only when simulcast is explicitly enabled via
  565. // config.js for VP9 and AV1 since full SVC is the default mode for these 2 codecs.
  566. || (this.codecSettings[codec].scalabilityModeEnabled
  567. && (codec === CodecMimeType.H264 || this.codecSettings[codec].useSimulcast));
  568. }
  569. /**
  570. * Munges the session description to ensure that the codec order is as per the preferred codec settings.
  571. *
  572. * @param {RTCSessionDescription} description - the local/remote description to be munged.
  573. * @returns {RTCSessionDescription} - the munged local/remote description.
  574. */
  575. mungeCodecOrder(description) {
  576. const codecSettings = this.pc.codecSettings;
  577. if (!codecSettings) {
  578. return description;
  579. }
  580. const { isP2P } = this.options;
  581. const parsedSdp = transform.parse(description.sdp);
  582. const mLines = parsedSdp.media.filter(m => m.type === codecSettings.mediaType);
  583. if (!mLines.length) {
  584. return description;
  585. }
  586. for (const mLine of mLines) {
  587. const currentCodecs = this.getConfiguredVideoCodecs(description);
  588. for (const codec of currentCodecs) {
  589. if (isP2P) {
  590. // 1. Strip the high profile H264 codecs on all clients. macOS started offering encoder for H.264
  591. // level 5.2 but a decoder only for level 3.1. Therfore, strip all main and high level codecs for
  592. // H.264.
  593. // 2. There are multiple VP9 payload types generated by the browser, more payload types are added
  594. // if the endpoint doesn't have a local video source. Therefore, strip all the high profile codec
  595. // variants for VP9 so that only one payload type for VP9 is negotiated between the peers.
  596. if (codec === CodecMimeType.H264 || codec === CodecMimeType.VP9) {
  597. SDPUtil.stripCodec(mLine, codec, true /* high profile */);
  598. }
  599. // Do not negotiate ULPFEC and RED either.
  600. if (codec === CodecMimeType.ULPFEC || codec === CodecMimeType.RED) {
  601. SDPUtil.stripCodec(mLine, codec, false);
  602. }
  603. }
  604. }
  605. // Reorder the codecs based on the preferred settings.
  606. if (!this.pc.usesCodecSelectionAPI()) {
  607. for (const codec of codecSettings.codecList.slice().reverse()) {
  608. SDPUtil.preferCodec(mLine, codec, isP2P);
  609. }
  610. }
  611. }
  612. return new RTCSessionDescription({
  613. type: description.type,
  614. sdp: transform.write(parsedSdp)
  615. });
  616. }
  617. /**
  618. * Munges the stereo flag as well as the opusMaxAverageBitrate in the SDP, based on values set through config.js,
  619. * if present.
  620. *
  621. * @param {RTCSessionDescription} description that needs to be munged.
  622. * @returns {RTCSessionDescription} the munged description.
  623. */
  624. mungeOpus(description) {
  625. const { audioQuality } = this.options;
  626. if (!audioQuality?.enableOpusDtx && !audioQuality?.stereo && !audioQuality?.opusMaxAverageBitrate) {
  627. return description;
  628. }
  629. const parsedSdp = transform.parse(description.sdp);
  630. const mLines = parsedSdp.media.filter(m => m.type === MediaType.AUDIO);
  631. for (const mLine of mLines) {
  632. const { payload } = mLine.rtp.find(protocol => protocol.codec === CodecMimeType.OPUS);
  633. if (!payload) {
  634. // eslint-disable-next-line no-continue
  635. continue;
  636. }
  637. let fmtpOpus = mLine.fmtp.find(protocol => protocol.payload === payload);
  638. if (!fmtpOpus) {
  639. fmtpOpus = {
  640. payload,
  641. config: ''
  642. };
  643. }
  644. const fmtpConfig = transform.parseParams(fmtpOpus.config);
  645. let sdpChanged = false;
  646. if (audioQuality?.stereo) {
  647. fmtpConfig.stereo = 1;
  648. sdpChanged = true;
  649. }
  650. if (audioQuality?.opusMaxAverageBitrate) {
  651. fmtpConfig.maxaveragebitrate = audioQuality.opusMaxAverageBitrate;
  652. sdpChanged = true;
  653. }
  654. // On Firefox, the OpusDtx enablement has no effect
  655. if (!browser.isFirefox() && audioQuality?.enableOpusDtx) {
  656. fmtpConfig.usedtx = 1;
  657. sdpChanged = true;
  658. }
  659. if (!sdpChanged) {
  660. // eslint-disable-next-line no-continue
  661. continue;
  662. }
  663. let mungedConfig = '';
  664. for (const key of Object.keys(fmtpConfig)) {
  665. mungedConfig += `${key}=${fmtpConfig[key]}; `;
  666. }
  667. fmtpOpus.config = mungedConfig.trim();
  668. }
  669. return new RTCSessionDescription({
  670. type: description.type,
  671. sdp: transform.write(parsedSdp)
  672. });
  673. }
  674. /**
  675. * Munges the session description by setting the max bitrates on the video m-lines when VP9 K-SVC codec is in use.
  676. *
  677. * @param {RTCSessionDescription} description - The local/remote description that needs to be munged.
  678. * @param {boolean} isLocalSdp - Whether the max bitrate (via b=AS line in SDP) is set on local SDP.
  679. * @returns {RTCSessionDescription} - The munged local/remote description.
  680. */
  681. setMaxBitrates(description, isLocalSdp = false) {
  682. const pcCodecSettings = this.pc.codecSettings;
  683. if (!pcCodecSettings) {
  684. return description;
  685. }
  686. const parsedSdp = transform.parse(description.sdp);
  687. // Find all the m-lines associated with the local sources.
  688. const direction = isLocalSdp ? MediaDirection.RECVONLY : MediaDirection.SENDONLY;
  689. const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO && m.direction !== direction);
  690. const currentCodec = pcCodecSettings.codecList[0];
  691. const codecScalabilityModeSettings = this.codecSettings[currentCodec];
  692. for (const mLine of mLines) {
  693. const isDoingVp9KSvc = currentCodec === CodecMimeType.VP9
  694. && !codecScalabilityModeSettings.scalabilityModeEnabled;
  695. const localTrack = this.pc.getLocalVideoTracks()
  696. .find(track => this.pc.localTrackTransceiverMids.get(track.rtcId) === mLine.mid.toString());
  697. if (localTrack
  698. && (isDoingVp9KSvc
  699. // Setting bitrates in the SDP for SVC codecs is no longer needed in the newer versions where
  700. // maxBitrates from the RTCRtpEncodingParameters directly affect the target bitrate for the encoder.
  701. || (this._isRunningInFullSvcMode(currentCodec) && !this.pc.usesCodecSelectionAPI()))) {
  702. let maxBitrate;
  703. if (localTrack.getVideoType() === VideoType.DESKTOP) {
  704. maxBitrate = codecScalabilityModeSettings.maxBitratesVideo.ssHigh;
  705. } else {
  706. const { level } = VIDEO_QUALITY_LEVELS.find(lvl => lvl.height <= localTrack.getCaptureResolution());
  707. maxBitrate = codecScalabilityModeSettings.maxBitratesVideo[level];
  708. }
  709. const limit = Math.floor(maxBitrate / 1000);
  710. // Use only the highest spatial layer bitrates for now as there is no API available yet for configuring
  711. // the bitrates on the individual SVC layers.
  712. mLine.bandwidth = [ {
  713. type: 'AS',
  714. limit
  715. } ];
  716. } else {
  717. // Clear the bandwidth limit in SDP when VP9 is no longer the preferred codec.
  718. // This is needed on react native clients as react-native-webrtc returns the
  719. // SDP that the application passed instead of returning the SDP off the native side.
  720. // This line automatically gets cleared on web on every renegotiation.
  721. mLine.bandwidth = undefined;
  722. }
  723. }
  724. return new RTCSessionDescription({
  725. type: description.type,
  726. sdp: transform.write(parsedSdp)
  727. });
  728. }
  729. /**
  730. * Checks if the AV1 Dependency descriptors are negotiated on the bridge peerconnection and removes them from the
  731. * description when codec selected is VP8 or VP9.
  732. *
  733. * @param {RTCSessionDescription} description that needs to be munged.
  734. * @returns {RTCSessionDescription} the munged description.
  735. */
  736. updateAv1DdHeaders(description) {
  737. const parsedSdp = transform.parse(description.sdp);
  738. const mLines = parsedSdp.media.filter(m => m.type === MediaType.VIDEO);
  739. if (!mLines.length || !browser.supportsDDExtHeaders()) {
  740. return description;
  741. }
  742. mLines.forEach((mLine, idx) => {
  743. const senderMids = Array.from(this.pc.localTrackTransceiverMids.values());
  744. const isSender = senderMids.length
  745. ? senderMids.find(mid => mLine.mid.toString() === mid.toString())
  746. : idx === 0;
  747. const payload = mLine.payloads.split(' ')[0];
  748. let { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload));
  749. codec = codec.toLowerCase();
  750. if (isSender && mLine.ext?.length) {
  751. const headerIndex = mLine.ext.findIndex(ext => ext.uri === DD_HEADER_EXT_URI);
  752. const shouldNegotiateHeaderExts = codec === CodecMimeType.AV1 || codec === CodecMimeType.H264;
  753. if (!this.supportsDDHeaderExt && headerIndex >= 0) {
  754. this.supportsDDHeaderExt = true;
  755. }
  756. if (this.supportsDDHeaderExt && shouldNegotiateHeaderExts && headerIndex < 0) {
  757. mLine.ext.push({
  758. value: DD_HEADER_EXT_ID,
  759. uri: DD_HEADER_EXT_URI
  760. });
  761. } else if (!shouldNegotiateHeaderExts && headerIndex >= 0) {
  762. mLine.ext.splice(headerIndex, 1);
  763. }
  764. }
  765. });
  766. return new RTCSessionDescription({
  767. type: description.type,
  768. sdp: transform.write(parsedSdp)
  769. });
  770. }
  771. }