From 42a9f968d8597ba34a99c9fe68cace8600876112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Velad=20Galv=C3=A1n?= Date: Wed, 15 Mar 2023 12:13:34 +0100 Subject: [PATCH] feat(HLS): Get resolution from TS when load a Media Playlist (#5058) --- build/types/core | 1 + lib/hls/hls_parser.js | 5 +- lib/util/exp_golomb.js | 206 ++++++++++++++++++++++++++++++++++++ lib/util/ts_parser.js | 145 +++++++++++++++++++++++++ test/hls/hls_parser_unit.js | 27 +++++ 5 files changed, 382 insertions(+), 2 deletions(-) create mode 100644 lib/util/exp_golomb.js diff --git a/build/types/core b/build/types/core index bd6c2e2421..84410247db 100644 --- a/build/types/core +++ b/build/types/core @@ -75,6 +75,7 @@ +../../lib/util/ebml_parser.js +../../lib/util/error.js +../../lib/util/event_manager.js ++../../lib/util/exp_golomb.js +../../lib/util/fairplay_utils.js +../../lib/util/fake_event.js +../../lib/util/fake_event_target.js diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index e9f1b83027..bc9cd606ac 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -882,13 +882,14 @@ shaka.hls.HlsParser = class { return null; } const onlyAudio = hasAudio && !hasVideo; + const videoResolution = tsParser.getVideoResolution(); return { type: onlyAudio ? 'audio' : 'video', mimeType: 'video/mp2t', codecs: codecs.join(', '), language: null, - height: null, - width: null, + height: videoResolution.height, + width: videoResolution.width, channelCount: null, sampleRate: null, }; diff --git a/lib/util/exp_golomb.js b/lib/util/exp_golomb.js new file mode 100644 index 0000000000..0fad5ad761 --- /dev/null +++ b/lib/util/exp_golomb.js @@ -0,0 +1,206 @@ +/** + * @license + * Copyright Brightcove, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +goog.provide('shaka.util.ExpGolomb'); + +goog.require('shaka.util.DataViewReader'); + + +/** + * @summary + * Parser for exponential Golomb codes, a variable-bitwidth number encoding + * scheme used by h264. + * Based on https://github.com/videojs/mux.js/blob/main/lib/utils/exp-golomb.js + * + * @export + */ +shaka.util.ExpGolomb = class { + /** + * @param {!Uint8Array} data + */ + constructor(data) { + /** @private {!Uint8Array} */ + this.data_ = data; + + /** @private {number} */ + this.workingBytesAvailable_ = data.byteLength; + + // the current word being examined + /** @private {number} */ + this.workingWord_ = 0; + + // the number of bits left to examine in the current word + /** @private {number} */ + this.workingBitsAvailable_ = 0; + } + + /** + * Load the next word + * + * @private + */ + loadWord_() { + const position = this.data_.byteLength - this.workingBytesAvailable_; + const bytes = new Uint8Array(4); + const availableBytes = Math.min(4, this.workingBytesAvailable_); + + if (availableBytes === 0) { + return; + } + + bytes.set(this.data_.subarray(position, position + availableBytes)); + const dataView = new shaka.util.DataViewReader( + bytes, shaka.util.DataViewReader.Endianness.BIG_ENDIAN); + this.workingWord_ = dataView.readUint32(); + + // track the amount of data that has been processed + this.workingBitsAvailable_ = availableBytes * 8; + this.workingBytesAvailable_ -= availableBytes; + } + + /** + * Skip n bits + * + * @param {number} count + */ + skipBits(count) { + if (this.workingBitsAvailable_ <= count) { + count -= this.workingBitsAvailable_; + const skipBytes = Math.floor(count / 8); + count -= (skipBytes * 8); + this.workingBitsAvailable_ -= skipBytes; + this.loadWord_(); + } + this.workingWord_ <<= count; + this.workingBitsAvailable_ -= count; + } + + /** + * Read n bits + * + * @param {number} size + * @return {number} + */ + readBits(size) { + let bits = Math.min(this.workingBitsAvailable_, size); + const valu = this.workingWord_ >>> (32 - bits); + this.workingBitsAvailable_ -= bits; + if (this.workingBitsAvailable_ > 0) { + this.workingWord_ <<= bits; + } else if (this.workingBytesAvailable_ > 0) { + this.loadWord_(); + } + bits = size - bits; + if (bits > 0) { + return (valu << bits) | this.readBits(bits); + } + return valu; + } + + /** + * Return the number of skip leading zeros + * + * @return {number} + * @private + */ + skipLeadingZeros_() { + let i; + for (i = 0; i < this.workingBitsAvailable_; ++i) { + if ((this.workingWord_ & (0x80000000 >>> i)) !== 0) { + // the first bit of working word is 1 + this.workingWord_ <<= i; + this.workingBitsAvailable_ -= i; + return i; + } + } + + // we exhausted workingWord and still have not found a 1 + this.loadWord_(); + return i + this.skipLeadingZeros_(); + } + + /** + * Skip exponential Golomb + */ + skipExpGolomb() { + this.skipBits(1 + this.skipLeadingZeros_()); + } + + /** + * Return unsigned exponential Golomb + * + * @return {number} + */ + readUnsignedExpGolomb() { + const clz = this.skipLeadingZeros_(); + return this.readBits(clz + 1) - 1; + } + + /** + * Return exponential Golomb + * + * @return {number} + */ + readExpGolomb() { + const valu = this.readUnsignedExpGolomb(); + if (0x01 & valu) { + // the number is odd if the low order bit is set + // add 1 to make it even, and divide by 2 + return (1 + valu) >>> 1; + } + // divide by two then make it negative + return -1 * (valu >>> 1); + } + + /** + * Read 1 bit as boolean + * + * @return {boolean} + */ + readBoolean() { + return this.readBits(1) === 1; + } + + /** + * Read 8 bits + * + * @return {number} + */ + readUnsignedByte() { + return this.readBits(8); + } + + /** + * The scaling list is optionally transmitted as part of a Sequence Parameter + * Set (SPS). + * + * @param {number} count the number of entries in this scaling list + * @see Recommendation ITU-T H.264, Section 7.3.2.1.1.1 + */ + skipScalingList(count) { + let lastScale = 8; + let nextScale = 8; + + for (let j = 0; j < count; j++) { + if (nextScale !== 0) { + const deltaScale = this.readExpGolomb(); + nextScale = (lastScale + deltaScale + 256) % 256; + } + lastScale = (nextScale === 0) ? lastScale : nextScale; + } + } +}; diff --git a/lib/util/ts_parser.js b/lib/util/ts_parser.js index 16d08f8e26..dc547aa1ef 100644 --- a/lib/util/ts_parser.js +++ b/lib/util/ts_parser.js @@ -8,6 +8,7 @@ goog.provide('shaka.util.TsParser'); goog.require('goog.asserts'); goog.require('shaka.log'); +goog.require('shaka.util.ExpGolomb'); goog.require('shaka.util.Id3Utils'); goog.require('shaka.util.Uint8ArrayUtils'); @@ -531,6 +532,129 @@ shaka.util.TsParser = class { return nalus.reverse(); } + /** + * Return the video resolution + * + * @return {{height: ?string, width: ?string}} + */ + getVideoResolution() { + const TsParser = shaka.util.TsParser; + const resolution = { + height: null, + width: null, + }; + const videoNalus = this.getVideoNalus(); + if (!videoNalus.length) { + return resolution; + } + const spsNalu = videoNalus.find((nalu) => { + return nalu.type == TsParser.H264_NALU_TYPE_SPS_; + }); + if (!spsNalu) { + return resolution; + } + + const expGolombDecoder = new shaka.util.ExpGolomb(spsNalu.data); + // profile_idc + const profileIdc = expGolombDecoder.readUnsignedByte(); + // constraint_set[0-5]_flag + expGolombDecoder.readUnsignedByte(); + // level_idc u(8) + expGolombDecoder.readUnsignedByte(); + // seq_parameter_set_id + expGolombDecoder.skipExpGolomb(); + + // some profiles have more optional data we don't need + if (TsParser.PROFILES_WITH_OPTIONAL_SPS_DATA_.includes(profileIdc)) { + const chromaFormatIdc = expGolombDecoder.readUnsignedExpGolomb(); + if (chromaFormatIdc === 3) { + // separate_colour_plane_flag + expGolombDecoder.skipBits(1); + } + // bit_depth_luma_minus8 + expGolombDecoder.skipExpGolomb(); + // bit_depth_chroma_minus8 + expGolombDecoder.skipExpGolomb(); + // qpprime_y_zero_transform_bypass_flag + expGolombDecoder.skipBits(1); + // seq_scaling_matrix_present_flag + if (expGolombDecoder.readBoolean()) { + const scalingListCount = (chromaFormatIdc !== 3) ? 8 : 12; + for (let i = 0; i < scalingListCount; i++) { + // seq_scaling_list_present_flag[ i ] + if (expGolombDecoder.readBoolean()) { + if (i < 6) { + expGolombDecoder.skipScalingList(16); + } else { + expGolombDecoder.skipScalingList(64); + } + } + } + } + } + + // log2_max_frame_num_minus4 + expGolombDecoder.skipExpGolomb(); + const picOrderCntType = expGolombDecoder.readUnsignedExpGolomb(); + + if (picOrderCntType === 0) { + // log2_max_pic_order_cnt_lsb_minus4 + expGolombDecoder.readUnsignedExpGolomb(); + } else if (picOrderCntType === 1) { + // delta_pic_order_always_zero_flag + expGolombDecoder.skipBits(1); + // offset_for_non_ref_pic + expGolombDecoder.skipExpGolomb(); + // offset_for_top_to_bottom_field + expGolombDecoder.skipExpGolomb(); + const numRefFramesInPicOrderCntCycle = + expGolombDecoder.readUnsignedExpGolomb(); + for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) { + // offset_for_ref_frame[ i ] + expGolombDecoder.skipExpGolomb(); + } + } + + // max_num_ref_frames + expGolombDecoder.skipExpGolomb(); + // gaps_in_frame_num_value_allowed_flag + expGolombDecoder.skipBits(1); + + const picWidthInMbsMinus1 = + expGolombDecoder.readUnsignedExpGolomb(); + const picHeightInMapUnitsMinus1 = + expGolombDecoder.readUnsignedExpGolomb(); + + const frameMbsOnlyFlag = expGolombDecoder.readBits(1); + if (frameMbsOnlyFlag === 0) { + // mb_adaptive_frame_field_flag + expGolombDecoder.skipBits(1); + } + // direct_8x8_inference_flag + expGolombDecoder.skipBits(1); + + let frameCropLeftOffset = 0; + let frameCropRightOffset = 0; + let frameCropTopOffset = 0; + let frameCropBottomOffset = 0; + + // frame_cropping_flag + if (expGolombDecoder.readBoolean()) { + frameCropLeftOffset = expGolombDecoder.readUnsignedExpGolomb(); + frameCropRightOffset = expGolombDecoder.readUnsignedExpGolomb(); + frameCropTopOffset = expGolombDecoder.readUnsignedExpGolomb(); + frameCropBottomOffset = expGolombDecoder.readUnsignedExpGolomb(); + } + + resolution.height = String(((2 - frameMbsOnlyFlag) * + (picHeightInMapUnitsMinus1 + 1) * 16) - (frameCropTopOffset * 2) - + (frameCropBottomOffset * 2)); + resolution.width = String(((picWidthInMbsMinus1 + 1) * 16) - + frameCropLeftOffset * 2 - frameCropRightOffset * 2); + + return resolution; + } + /** * Check if the passed data corresponds to an MPEG2-TS * @@ -591,6 +715,27 @@ shaka.util.TsParser.PacketLength_ = 188; shaka.util.TsParser.Timescale_ = 90000; +/** + * NALU type for Sequence Parameter Set (SPS) for H.264. + * @const {number} + * @private + */ +shaka.util.TsParser.H264_NALU_TYPE_SPS_ = 0x07; + + +/** + * Values of profile_idc that indicate additional fields are included in the + * SPS. + * see Recommendation ITU-T H.264 (4/2013) + * 7.3.2.1.1 Sequence parameter set data syntax + * + * @const {!Array.} + * @private + */ +shaka.util.TsParser.PROFILES_WITH_OPTIONAL_SPS_DATA_ = + [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134]; + + /** * @typedef {{ * audio: number, diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js index 70fa06083f..ec3fe3e74b 100644 --- a/test/hls/hls_parser_unit.js +++ b/test/hls/hls_parser_unit.js @@ -12,6 +12,7 @@ describe('HlsParser', () => { const videoInitSegmentUri = '/base/test/test/assets/sintel-video-init.mp4'; const videoSegmentUri = '/base/test/test/assets/sintel-video-segment.mp4'; + const videoTsSegmentUri = '/base/test/test/assets/video.ts'; const vttText = [ 'WEBVTT\n', @@ -37,6 +38,8 @@ describe('HlsParser', () => { /** @type {!Uint8Array} */ let segmentData; /** @type {!Uint8Array} */ + let tsSegmentData; + /** @type {!Uint8Array} */ let selfInitializingSegmentData; /** @type {!Uint8Array} */ let aes128Key; @@ -52,6 +55,7 @@ describe('HlsParser', () => { const responses = await Promise.all([ shaka.test.Util.fetch(videoInitSegmentUri), shaka.test.Util.fetch(videoSegmentUri), + shaka.test.Util.fetch(videoTsSegmentUri), ]); initSegmentData = responses[0]; segmentData = responses[1]; @@ -59,6 +63,8 @@ describe('HlsParser', () => { selfInitializingSegmentData = shaka.util.Uint8ArrayUtils.concat(initSegmentData, segmentData); + tsSegmentData = responses[2]; + aes128Key = new Uint8Array([ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, @@ -4468,4 +4474,25 @@ describe('HlsParser', () => { expect(actualAudio1.mimeType).toBe('audio/aac'); expect(actualAudio1.codecs).toBe(''); }); + + it('parses media playlists directly', async () => { + const mediaPlaylist = [ + '#EXTM3U\n', + '#EXT-X-TARGETDURATION:5\n', + '#EXTINF:5,\n', + 'video1.ts\n', + ].join(''); + + fakeNetEngine + .setResponseText('test:/master', mediaPlaylist) + .setResponseValue('test:/video1.ts', tsSegmentData); + + const actualManifest = await parser.start('test:/master', playerInterface); + expect(actualManifest.variants.length).toBe(1); + + const video = actualManifest.variants[0].video; + + expect(video.width).toBe(256); + expect(video.height).toBe(110); + }); });