Source: lib/transmuxer/h264.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.transmuxer.H264');
  9. goog.require('shaka.util.ExpGolomb');
  10. goog.require('shaka.util.Uint8ArrayUtils');
  13. /**
  14.  * H.264 utils
  15.  */
  16. shaka.transmuxer.H264 = class {
  17.   /**
  18.    * Read a sequence parameter set and return some interesting video
  19.    * properties. A sequence parameter set is the H264 metadata that
  20.    * describes the properties of upcoming video frames.
  21.    *
  22.    * @param {!Array.<shaka.extern.VideoNalu>} nalus
  23.    * @return {?{height: number, width: number, videoConfig: !Uint8Array,
  24.    *            hSpacing: number, vSpacing: number}}
  25.    */
  26.   static parseInfo(nalus) {
  27.     const H264 = shaka.transmuxer.H264;
  28.     if (!nalus.length) {
  29.       return null;
  30.     }
  31.     const spsNalu = nalus.find((nalu) => {
  32.       return nalu.type == H264.NALU_TYPE_SPS_;
  33.     });
  34.     const ppsNalu = nalus.find((nalu) => {
  35.       return nalu.type == H264.NALU_TYPE_PPS_;
  36.     });
  37.     if (!spsNalu || !ppsNalu) {
  38.       return null;
  39.     }
  41.     const expGolombDecoder = new shaka.util.ExpGolomb(spsNalu.data);
  42.     // profile_idc
  43.     const profileIdc = expGolombDecoder.readUnsignedByte();
  44.     // constraint_set[0-5]_flag
  45.     expGolombDecoder.readUnsignedByte();
  46.     // level_idc u(8)
  47.     expGolombDecoder.readUnsignedByte();
  48.     // seq_parameter_set_id
  49.     expGolombDecoder.skipExpGolomb();
  51.     // some profiles have more optional data we don't need
  52.     if (H264.PROFILES_WITH_OPTIONAL_SPS_DATA_.includes(profileIdc)) {
  53.       const chromaFormatIdc = expGolombDecoder.readUnsignedExpGolomb();
  54.       if (chromaFormatIdc === 3) {
  55.         // separate_colour_plane_flag
  56.         expGolombDecoder.skipBits(1);
  57.       }
  58.       // bit_depth_luma_minus8
  59.       expGolombDecoder.skipExpGolomb();
  60.       // bit_depth_chroma_minus8
  61.       expGolombDecoder.skipExpGolomb();
  62.       // qpprime_y_zero_transform_bypass_flag
  63.       expGolombDecoder.skipBits(1);
  64.       // seq_scaling_matrix_present_flag
  65.       if (expGolombDecoder.readBoolean()) {
  66.         const scalingListCount = (chromaFormatIdc !== 3) ? 8 : 12;
  67.         for (let i = 0; i < scalingListCount; i++) {
  68.           // seq_scaling_list_present_flag[ i ]
  69.           if (expGolombDecoder.readBoolean()) {
  70.             if (i < 6) {
  71.               expGolombDecoder.skipScalingList(16);
  72.             } else {
  73.               expGolombDecoder.skipScalingList(64);
  74.             }
  75.           }
  76.         }
  77.       }
  78.     }
  80.     // log2_max_frame_num_minus4
  81.     expGolombDecoder.skipExpGolomb();
  82.     const picOrderCntType = expGolombDecoder.readUnsignedExpGolomb();
  84.     if (picOrderCntType === 0) {
  85.       // log2_max_pic_order_cnt_lsb_minus4
  86.       expGolombDecoder.readUnsignedExpGolomb();
  87.     } else if (picOrderCntType === 1) {
  88.       // delta_pic_order_always_zero_flag
  89.       expGolombDecoder.skipBits(1);
  90.       // offset_for_non_ref_pic
  91.       expGolombDecoder.skipExpGolomb();
  92.       // offset_for_top_to_bottom_field
  93.       expGolombDecoder.skipExpGolomb();
  94.       const numRefFramesInPicOrderCntCycle =
  95.           expGolombDecoder.readUnsignedExpGolomb();
  96.       for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
  97.         // offset_for_ref_frame[ i ]
  98.         expGolombDecoder.skipExpGolomb();
  99.       }
  100.     }
  102.     // max_num_ref_frames
  103.     expGolombDecoder.skipExpGolomb();
  104.     // gaps_in_frame_num_value_allowed_flag
  105.     expGolombDecoder.skipBits(1);
  107.     const picWidthInMbsMinus1 =
  108.         expGolombDecoder.readUnsignedExpGolomb();
  109.     const picHeightInMapUnitsMinus1 =
  110.         expGolombDecoder.readUnsignedExpGolomb();
  112.     const frameMbsOnlyFlag = expGolombDecoder.readBits(1);
  113.     if (frameMbsOnlyFlag === 0) {
  114.       // mb_adaptive_frame_field_flag
  115.       expGolombDecoder.skipBits(1);
  116.     }
  117.     // direct_8x8_inference_flag
  118.     expGolombDecoder.skipBits(1);
  120.     let frameCropLeftOffset = 0;
  121.     let frameCropRightOffset = 0;
  122.     let frameCropTopOffset = 0;
  123.     let frameCropBottomOffset = 0;
  125.     // frame_cropping_flag
  126.     if (expGolombDecoder.readBoolean()) {
  127.       frameCropLeftOffset = expGolombDecoder.readUnsignedExpGolomb();
  128.       frameCropRightOffset = expGolombDecoder.readUnsignedExpGolomb();
  129.       frameCropTopOffset = expGolombDecoder.readUnsignedExpGolomb();
  130.       frameCropBottomOffset = expGolombDecoder.readUnsignedExpGolomb();
  131.     }
  133.     let hSpacing = 1;
  134.     let vSpacing = 1;
  136.     // vui_parameters_present_flag
  137.     if (expGolombDecoder.readBoolean()) {
  138.       // aspect_ratio_info_present_flag
  139.       if (expGolombDecoder.readBoolean()) {
  140.         const aspectRatioIdc = expGolombDecoder.readUnsignedByte();
  141.         const hSpacingTable = [
  142.           1, 12, 10, 16, 40, 24, 20, 32, 80, 18, 15, 64, 160, 4, 3, 2,
  143.         ];
  144.         const vSpacingTable = [
  145.           1, 11, 11, 11, 33, 11, 11, 11, 33, 11, 11, 33, 99, 3, 2, 1,
  146.         ];
  147.         if (aspectRatioIdc > 0 && aspectRatioIdc <= 16) {
  148.           hSpacing = hSpacingTable[aspectRatioIdc - 1];
  149.           vSpacing = vSpacingTable[aspectRatioIdc - 1];
  150.         } else if (aspectRatioIdc === 255) {
  151.           hSpacing = expGolombDecoder.readBits(16);
  152.           vSpacing = expGolombDecoder.readBits(16);
  153.         }
  154.       }
  155.     }
  157.     const height = ((2 - frameMbsOnlyFlag) *
  158.         (picHeightInMapUnitsMinus1 + 1) * 16) - (frameCropTopOffset * 2) -
  159.         (frameCropBottomOffset * 2);
  160.     const width = ((picWidthInMbsMinus1 + 1) * 16) -
  161.         frameCropLeftOffset * 2 - frameCropRightOffset * 2;
  163.     // assemble the SPSs
  164.     let sps = [];
  165.     const spsData = spsNalu.fullData;
  166.     sps.push((spsData.byteLength >>> 8) & 0xff);
  167.     sps.push(spsData.byteLength & 0xff);
  168.     sps = sps.concat(...spsData);
  170.     // assemble the PPSs
  171.     let pps = [];
  172.     const ppsData = ppsNalu.fullData;
  173.     pps.push((ppsData.byteLength >>> 8) & 0xff);
  174.     pps.push(ppsData.byteLength & 0xff);
  175.     pps = pps.concat(...ppsData);
  177.     const videoConfig = new Uint8Array(
  178.         [
  179.           0x01, // version
  180.           sps[3], // profile
  181.           sps[4], // profile compat
  182.           sps[5], // level
  183.           0xfc | 3, // lengthSizeMinusOne, hard-coded to 4 bytes
  184.           0xe0 | 1, // 3bit reserved (111) + numOfSequenceParameterSets
  185.         ].concat(sps).concat([
  186.           1, // numOfPictureParameterSets
  187.         ]).concat(pps));
  189.     return {
  190.       height,
  191.       width,
  192.       videoConfig,
  193.       hSpacing,
  194.       vSpacing,
  195.     };
  196.   }
  198.   /**
  199.    * @param {!Array.<shaka.extern.VideoNalu>} nalus
  200.    * @return {?{data: !Uint8Array, isKeyframe: boolean}}
  201.    */
  202.   static parseFrame(nalus) {
  203.     const H264 = shaka.transmuxer.H264;
  204.     let isKeyframe = false;
  205.     const nalusData = [];
  206.     const spsNalu = nalus.find((nalu) => {
  207.       return nalu.type == H264.NALU_TYPE_SPS_;
  208.     });
  209.     let avcSample = false;
  210.     for (const nalu of nalus) {
  211.       let push = false;
  212.       switch (nalu.type) {
  213.         case H264.NALU_TYPE_NDR_: {
  214.           avcSample = true;
  215.           push = true;
  216.           const data = nalu.data;
  217.           // Only check slice type to detect KF in case SPS found in same packet
  218.           // (any keyframe is preceded by SPS ...)
  219.           if (spsNalu && data.length > 4) {
  220.             // retrieve slice type by parsing beginning of NAL unit (follow
  221.             // H264 spec,slice_header definition) to detect keyframe embedded
  222.             // in NDR
  223.             const sliceType = new shaka.util.ExpGolomb(data).readSliceType();
  224.             // 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice
  225.             // SI slice : A slice that is coded using intra prediction only and
  226.             // using quantisation of the prediction samples.
  227.             // An SI slice can be coded such that its decoded samples can be
  228.             // constructed identically to an SP slice.
  229.             // I slice: A slice that is not an SI slice that is decoded using
  230.             // intra prediction only.
  231.             if (sliceType === 2 || sliceType === 4 ||
  232.                 sliceType === 7 || sliceType === 9) {
  233.               isKeyframe = true;
  234.             }
  235.           }
  236.           break;
  237.         }
  238.         case H264.NALU_TYPE_IDR_:
  239.           avcSample = true;
  240.           push = true;
  241.           isKeyframe = true;
  242.           break;
  243.         case H264.NALU_TYPE_SEI_:
  244.           push = true;
  245.           break;
  246.         case H264.NALU_TYPE_SPS_:
  247.           push = true;
  248.           break;
  249.         case H264.NALU_TYPE_PPS_:
  250.           push = true;
  251.           break;
  252.         case H264.NALU_TYPE_AUD_:
  253.           push = true;
  254.           avcSample = true;
  255.           break;
  256.         case H264.NALU_TYPE_FILLER_DATA_:
  257.           push = true;
  258.           break;
  259.         default:
  260.           push = false;
  261.           break;
  262.       }
  263.       if (avcSample && push) {
  264.         const size = nalu.fullData.byteLength;
  265.         const naluLength = new Uint8Array(4);
  266.         naluLength[0] = (size >> 24) & 0xff;
  267.         naluLength[1] = (size >> 16) & 0xff;
  268.         naluLength[2] = (size >> 8) & 0xff;
  269.         naluLength[3] = size & 0xff;
  270.         nalusData.push(naluLength);
  271.         nalusData.push(nalu.fullData);
  272.       }
  273.     }
  274.     if (!nalusData.length) {
  275.       return null;
  276.     }
  277.     const data = shaka.util.Uint8ArrayUtils.concat(...nalusData);
  278.     return {
  279.       data,
  280.       isKeyframe,
  281.     };
  282.   }
  283. };
  286. /**
  287.  * NALU type for NDR for H.264.
  288.  * @const {number}
  289.  * @private
  290.  */
  291. shaka.transmuxer.H264.NALU_TYPE_NDR_ = 0x01;
  294. /**
  295.  * NALU type for Instantaneous Decoder Refresh (IDR) for H.264.
  296.  * @const {number}
  297.  * @private
  298.  */
  299. shaka.transmuxer.H264.NALU_TYPE_IDR_ = 0x05;
  302. /**
  303.  * NALU type for Supplemental Enhancement Information (SEI) for H.264.
  304.  * @const {number}
  305.  * @private
  306.  */
  307. shaka.transmuxer.H264.NALU_TYPE_SEI_ = 0x06;
  310. /**
  311.  * NALU type for Sequence Parameter Set (SPS) for H.264.
  312.  * @const {number}
  313.  * @private
  314.  */
  315. shaka.transmuxer.H264.NALU_TYPE_SPS_ = 0x07;
  318. /**
  319.  * NALU type for Picture Parameter Set (PPS) for H.264.
  320.  * @const {number}
  321.  * @private
  322.  */
  323. shaka.transmuxer.H264.NALU_TYPE_PPS_ = 0x08;
  326. /**
  327.  * NALU type for Access Unit Delimiter (AUD) for H.264.
  328.  * @const {number}
  329.  * @private
  330.  */
  331. shaka.transmuxer.H264.NALU_TYPE_AUD_ = 0x09;
  334. /**
  335.  * NALU type for Filler Data for H.264.
  336.  * @const {number}
  337.  * @private
  338.  */
  339. shaka.transmuxer.H264.NALU_TYPE_FILLER_DATA_ = 0x0c;
  342. /**
  343.  * Values of profile_idc that indicate additional fields are included in the
  344.  * SPS.
  345.  * see Recommendation ITU-T H.264 (4/2013)
  346.  * 7.3.2.1.1 Sequence parameter set data syntax
  347.  *
  348.  * @const {!Array.<number>}
  349.  * @private
  350.  */
  351. shaka.transmuxer.H264.PROFILES_WITH_OPTIONAL_SPS_DATA_ =
  352.     [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134];