From 264c84249684ee809f53fd4117f9aab4e0a599ac Mon Sep 17 00:00:00 2001 From: Theodore Abshire Date: Mon, 10 Jan 2022 15:36:59 -0800 Subject: [PATCH] fix(image): Fix HLS image track issues This makes the HLS parser honor more attributes for image tracks. It also makes some changes to player.getImageTracks, so that the returned track shows the size of a single thumbnail rather than the entire sheet. Closes #3840 Change-Id: I2ae096f455864201e08a85e29f0f02a3e06eb07f --- lib/hls/hls_parser.js | 30 ++++++++++++++++++++++++++++++ lib/media/segment_index.js | 3 ++- lib/media/segment_reference.js | 20 +++++++++++++++++++- lib/player.js | 3 ++- lib/util/stream_utils.js | 23 +++++++++++++++++++++-- test/player_unit.js | 10 ++++++---- 6 files changed, 80 insertions(+), 9 deletions(-) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index 0aa224f697..42de5a8d21 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -1281,6 +1281,30 @@ shaka.hls.HlsParser = class { if (this.uriToStreamInfosMap_.has(verbatimImagePlaylistUri)) { return this.uriToStreamInfosMap_.get(verbatimImagePlaylistUri); } + + // Parse misc attributes. + const resolution = tag.getAttributeValue('RESOLUTION'); + if (resolution) { + // The RESOLUTION tag represents the resolution of a single thumbnail, not + // of the entire sheet at once (like we expect in the output). + // So multiply by the layout size. + + const reference = streamInfo.stream.segmentIndex.get(0); + const layout = reference.getTilesLayout(); + if (layout) { + streamInfo.stream.width = + Number(resolution.split('x')[0]) * Number(layout.split('x')[0]); + streamInfo.stream.height = + Number(resolution.split('x')[1]) * Number(layout.split('x')[1]); + // TODO: What happens if there are multiple grids, with different + // layout sizes, inside this image stream? + } + } + const bandwidth = tag.getAttributeValue('BANDWIDTH'); + if (bandwidth) { + streamInfo.stream.bandwidth = Number(bandwidth); + } + this.uriToStreamInfosMap_.set(verbatimImagePlaylistUri, streamInfo); return streamInfo; } @@ -1814,6 +1838,7 @@ shaka.hls.HlsParser = class { } let tilesLayout = ''; + let tileDuration = null; if (type == shaka.util.ManifestParserUtils.ContentType.IMAGE) { // By default in HLS the tilesLayout is 1x1 tilesLayout = '1x1'; @@ -1821,6 +1846,10 @@ shaka.hls.HlsParser = class { shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-TILES'); if (tilesTag) { tilesLayout = tilesTag.getRequiredAttrValue('LAYOUT'); + const duration = tilesTag.getAttributeValue('DURATION'); + if (duration) { + tileDuration = Number(duration); + } } } @@ -1836,6 +1865,7 @@ shaka.hls.HlsParser = class { /* appendWindowEnd= */ Infinity, partialSegmentRefs, tilesLayout, + tileDuration, ); } diff --git a/lib/media/segment_index.js b/lib/media/segment_index.js index 8c167d4a90..6147b80abe 100644 --- a/lib/media/segment_index.js +++ b/lib/media/segment_index.js @@ -347,7 +347,8 @@ shaka.media.SegmentIndex = class { lastReference.appendWindowStart, lastReference.appendWindowEnd, lastReference.partialReferences, - lastReference.tilesLayout); + lastReference.tilesLayout, + lastReference.tileDuration); } diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index a11f6b6de7..ab146015aa 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -155,11 +155,15 @@ shaka.media.SegmentReference = class { * The value is a grid-item-dimension consisting of two positive decimal * integers in the format: column-x-row ('4x3'). It describes the * arrangement of Images in a Grid. The minimum valid LAYOUT is '1x1'. + * @param {?number=} tileDuration + * The explicit duration of an individual tile within the tiles grid. + * If not provided, the duration should be automatically calculated based on + * the duration of the reference. */ constructor( startTime, endTime, uris, startByte, endByte, initSegmentReference, timestampOffset, appendWindowStart, appendWindowEnd, - partialReferences = [], tilesLayout = '') { + partialReferences = [], tilesLayout = '', tileDuration = null) { // A preload hinted Partial Segment has the same startTime and endTime. goog.asserts.assert(startTime <= endTime, 'startTime must be less than or equal to endTime'); @@ -206,6 +210,9 @@ shaka.media.SegmentReference = class { /** @type {?string} */ this.tilesLayout = tilesLayout; + + /** @type {?number} */ + this.tileDuration = tileDuration; } /** @@ -290,6 +297,17 @@ shaka.media.SegmentReference = class { getTilesLayout() { return this.tilesLayout; } + + /** + * Returns the segment's explicit tile duration. + * Only defined in image segments. + * + * @return {?number} + * @export + */ + getTileDuration() { + return this.tileDuration; + } }; diff --git a/lib/player.js b/lib/player.js index feb66dee68..964c67c609 100644 --- a/lib/player.js +++ b/lib/player.js @@ -3613,7 +3613,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { const height = fullImageHeight / rows; const totalImages = columns * rows; const segmentDuration = reference.trueEndTime - reference.startTime; - const thumbnailDuration = segmentDuration / totalImages; + const thumbnailDuration = + reference.getTileDuration() || (segmentDuration / totalImages); let thumbnailTime = reference.startTime; let positionX = 0; let positionY = 0; diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index 67e1396d6c..f8358d68ff 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -989,6 +989,25 @@ shaka.util.StreamUtils = class { static imageStreamToTrack(stream) { const ContentType = shaka.util.ManifestParserUtils.ContentType; + let width = stream.width || null; + let height = stream.height || null; + + // The stream width and height represent the size of the entire thumbnail + // sheet, so divide by the layout. + const reference = stream.segmentIndex.get(0); + let layout = stream.tilesLayout; + if (reference) { + layout = reference.getTilesLayout() || layout; + } + if (layout && width != null) { + width /= Number(layout.split('x')[0]); + } + if (layout && height != null) { + height /= Number(layout.split('x')[1]); + } + // TODO: What happens if there are multiple grids, with different + // layout sizes, inside this image stream? + /** @type {shaka.extern.Track} */ const track = { id: stream.id, @@ -998,8 +1017,8 @@ shaka.util.StreamUtils = class { language: '', label: null, kind: null, - width: stream.width || null, - height: stream.height || null, + width, + height, frameRate: null, pixelAspectRatio: null, hdr: null, diff --git a/test/player_unit.js b/test/player_unit.js index fc791d2dde..5d83ce3233 100644 --- a/test/player_unit.js +++ b/test/player_unit.js @@ -1092,8 +1092,8 @@ describe('Player', () => { // Image tracks manifest.addImageStream(53, (stream) => { stream.originalId = 'thumbnail'; - stream.width = 100; - stream.height = 200; + stream.width = 200; + stream.height = 400; stream.bandwidth = 10; stream.mimeType = 'image/jpeg'; stream.tilesLayout = '1x1'; @@ -1503,8 +1503,8 @@ describe('Player', () => { audioBandwidth: null, videoBandwidth: null, bandwidth: 10, - width: 100, - height: 200, + width: 200, + height: 400, frameRate: null, pixelAspectRatio: null, hdr: null, @@ -3537,6 +3537,8 @@ describe('Player', () => { await player.load(fakeManifestUri, 0, fakeMimeType); + expect(player.getImageTracks()[0].width).toBe(100); + expect(player.getImageTracks()[0].height).toBe(50); const thumbnail0 = await player.getThumbnails(5, 0); const thumbnail1 = await player.getThumbnails(5, 11); const thumbnail2 = await player.getThumbnails(5, 21);