From ca751fb0dd0b145389c25a45e1cb1e350dc20a05 Mon Sep 17 00:00:00 2001 From: Alvaro Velad Date: Wed, 12 Jan 2022 17:23:47 +0100 Subject: [PATCH 1/5] Allow WebP and AVIF image streams --- lib/util/stream_utils.js | 109 +++++++++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 22 deletions(-) diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index ca32535840..3fd0c5a897 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -406,7 +406,7 @@ shaka.util.StreamUtils = class { shaka.util.StreamUtils.filterManifestByCurrentVariant( currentVariant, manifest); shaka.util.StreamUtils.filterTextStreams_(manifest); - shaka.util.StreamUtils.filterImageStreams_(manifest); + await shaka.util.StreamUtils.filterImageStreams_(manifest); } @@ -761,37 +761,61 @@ shaka.util.StreamUtils = class { * @param {shaka.extern.Manifest} manifest * @private */ - static filterImageStreams_(manifest) { - // Filter image streams. - manifest.imageStreams = manifest.imageStreams.filter((stream) => { - // TODO: re-examine this and avoid allow-listing the MIME types we can - // accept. - const validMimeTypes = [ - 'image/svg+xml', - 'image/png', - 'image/jpeg', - ]; - const Platform = shaka.util.Platform; - // Add webp support to popular platforms that support it. - const webpSupport = Platform.isWebOS() || - Platform.isTizen() || - Platform.isChromecast(); - if (webpSupport) { - validMimeTypes.push('image/webp'); + static async filterImageStreams_(manifest) { + const imageStreams = []; + for (const stream of manifest.imageStreams) { + const mimeType = stream.mimeType; + if (!shaka.util.StreamUtils.supportedImageMimeTypes.has(mimeType) && + !shaka.util.StreamUtils.unsupportedImageMimeTypes.has(mimeType)) { + const minImage = shaka.util.StreamUtils.minImage.get(mimeType); + if (minImage) { + // eslint-disable-next-line no-await-in-loop + const res = await shaka.util.StreamUtils.isImageSupported_(minImage); + if (res) { + shaka.util.StreamUtils.supportedImageMimeTypes.add(mimeType); + } else { + shaka.util.StreamUtils.unsupportedImageMimeTypes.add(mimeType); + } + } else { + shaka.util.StreamUtils.unsupportedImageMimeTypes.add(mimeType); + } } - // TODO: add support to image/webp and image/avif - const keep = validMimeTypes.includes(stream.mimeType); + + const keep = shaka.util.StreamUtils.supportedImageMimeTypes.has(mimeType); if (!keep) { shaka.log.debug('Dropping image stream. Is not supported by the ' + 'platform.', stream); + } else { + imageStreams.push(stream); } + } + manifest.imageStreams = imageStreams; + } - return keep; + /** + * @param {string} minImage + * @return {!Promise.} + * @private + */ + static isImageSupported_(minImage) { + return new Promise((resolve) => { + const imageElement = /** @type {HTMLImageElement} */(new Image()); + imageElement.src = minImage; + if ('decode' in imageElement) { + imageElement.decode().then(() => { + resolve(true); + }).catch(() => { + resolve(false); + }); + } else { + imageElement.onload = imageElement.onerror = () => { + resolve(imageElement.height === 2); + }; + } }); } - /** * @param {shaka.extern.Stream} s0 * @param {shaka.extern.Stream} s1 @@ -1445,3 +1469,44 @@ shaka.util.StreamUtils.DecodingAttributes = { POWER: 'powerEfficient', BANDWIDTH: 'bandwidth', }; + +/** + * @private {!Set.} + */ +shaka.util.StreamUtils.supportedImageMimeTypes = new Set() + .add('image/svg+xml') + .add('image/png') + .add('image/jpeg'); + +/** + * @private {!Set.} + */ +shaka.util.StreamUtils.unsupportedImageMimeTypes = new Set(); + +/** + * @const {string} + * @private + */ +shaka.util.StreamUtils.minWebPImage = 'data:image/webp;base64,UklGRjoAAABXRU' + + 'JQVlA4IC4AAACyAgCdASoCAAIALmk0mk0iIiIiIgBoSygABc6WWgAA/veff/0PP8bA//LwY' + + 'AAA'; + +/** + * @const {string} + * @private + */ +shaka.util.StreamUtils.minAvifImage = 'data:image/avif;base64,AAAAIGZ0eXBhdm' + + 'lmAAAAAGF2aWZtaWYxbWlhZk1BMUIAAADybWV0YQAAAAAAAAAoaGRscgAAAAAAAAAAcGljd' + + 'AAAAAAAAAAAAAAAAGxpYmF2aWYAAAAADnBpdG0AAAAAAAEAAAAeaWxvYwAAAABEAAABAAEA' + + 'AAABAAABGgAAAB0AAAAoaWluZgAAAAAAAQAAABppbmZlAgAAAAABAABhdjAxQ29sb3IAAAA' + + 'AamlwcnAAAABLaXBjbwAAABRpc3BlAAAAAAAAAAIAAAACAAAAEHBpeGkAAAAAAwgICAAAAA' + + 'xhdjFDgQ0MAAAAABNjb2xybmNseAACAAIAAYAAAAAXaXBtYQAAAAAAAAABAAEEAQKDBAAAA' + + 'CVtZGF0EgAKCBgANogQEAwgMg8f8D///8WfhwB8+ErK42A='; + +/** + * @const {!Map.} + * @private + */ +shaka.util.StreamUtils.minImage = new Map() + .set('image/webp', shaka.util.StreamUtils.minWebPImage) + .set('image/avif', shaka.util.StreamUtils.minAvifImage); From 3ce05a30690405f001b919f777a76a4efb4b90c3 Mon Sep 17 00:00:00 2001 From: Alvaro Velad Date: Wed, 12 Jan 2022 18:09:31 +0100 Subject: [PATCH 2/5] Use underscore for private variables --- lib/util/stream_utils.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index 3fd0c5a897..7eb5f53744 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -765,23 +765,23 @@ shaka.util.StreamUtils = class { const imageStreams = []; for (const stream of manifest.imageStreams) { const mimeType = stream.mimeType; - if (!shaka.util.StreamUtils.supportedImageMimeTypes.has(mimeType) && - !shaka.util.StreamUtils.unsupportedImageMimeTypes.has(mimeType)) { + if (!shaka.util.StreamUtils.supportedImageMimeTypes_.has(mimeType) && + !shaka.util.StreamUtils.unsupportedImageMimeTypes_.has(mimeType)) { const minImage = shaka.util.StreamUtils.minImage.get(mimeType); if (minImage) { // eslint-disable-next-line no-await-in-loop const res = await shaka.util.StreamUtils.isImageSupported_(minImage); if (res) { - shaka.util.StreamUtils.supportedImageMimeTypes.add(mimeType); + shaka.util.StreamUtils.supportedImageMimeTypes_.add(mimeType); } else { - shaka.util.StreamUtils.unsupportedImageMimeTypes.add(mimeType); + shaka.util.StreamUtils.unsupportedImageMimeTypes_.add(mimeType); } } else { - shaka.util.StreamUtils.unsupportedImageMimeTypes.add(mimeType); + shaka.util.StreamUtils.unsupportedImageMimeTypes_.add(mimeType); } } - const keep = shaka.util.StreamUtils.supportedImageMimeTypes.has(mimeType); + const keep = shaka.util.StreamUtils.supportedImageMimeTypes_.has(mimeType); if (!keep) { shaka.log.debug('Dropping image stream. Is not supported by the ' + @@ -1473,7 +1473,7 @@ shaka.util.StreamUtils.DecodingAttributes = { /** * @private {!Set.} */ -shaka.util.StreamUtils.supportedImageMimeTypes = new Set() +shaka.util.StreamUtils.supportedImageMimeTypes_ = new Set() .add('image/svg+xml') .add('image/png') .add('image/jpeg'); @@ -1481,13 +1481,13 @@ shaka.util.StreamUtils.supportedImageMimeTypes = new Set() /** * @private {!Set.} */ -shaka.util.StreamUtils.unsupportedImageMimeTypes = new Set(); +shaka.util.StreamUtils.unsupportedImageMimeTypes_ = new Set(); /** * @const {string} * @private */ -shaka.util.StreamUtils.minWebPImage = 'data:image/webp;base64,UklGRjoAAABXRU' + +shaka.util.StreamUtils.minWebPImage_ = 'data:image/webp;base64,UklGRjoAAABXRU' + 'JQVlA4IC4AAACyAgCdASoCAAIALmk0mk0iIiIiIgBoSygABc6WWgAA/veff/0PP8bA//LwY' + 'AAA'; @@ -1495,7 +1495,7 @@ shaka.util.StreamUtils.minWebPImage = 'data:image/webp;base64,UklGRjoAAABXRU' + * @const {string} * @private */ -shaka.util.StreamUtils.minAvifImage = 'data:image/avif;base64,AAAAIGZ0eXBhdm' + +shaka.util.StreamUtils.minAvifImage_ = 'data:image/avif;base64,AAAAIGZ0eXBhdm' + 'lmAAAAAGF2aWZtaWYxbWlhZk1BMUIAAADybWV0YQAAAAAAAAAoaGRscgAAAAAAAAAAcGljd' + 'AAAAAAAAAAAAAAAAGxpYmF2aWYAAAAADnBpdG0AAAAAAAEAAAAeaWxvYwAAAABEAAABAAEA' + 'AAABAAABGgAAAB0AAAAoaWluZgAAAAAAAQAAABppbmZlAgAAAAABAABhdjAxQ29sb3IAAAA' + @@ -1508,5 +1508,5 @@ shaka.util.StreamUtils.minAvifImage = 'data:image/avif;base64,AAAAIGZ0eXBhdm' + * @private */ shaka.util.StreamUtils.minImage = new Map() - .set('image/webp', shaka.util.StreamUtils.minWebPImage) - .set('image/avif', shaka.util.StreamUtils.minAvifImage); + .set('image/webp', shaka.util.StreamUtils.minWebPImage_) + .set('image/avif', shaka.util.StreamUtils.minAvifImage_); From 5ab3be8ddb1da577cde470f2b1833ed00e8624f4 Mon Sep 17 00:00:00 2001 From: Alvaro Velad Date: Wed, 12 Jan 2022 18:10:43 +0100 Subject: [PATCH 3/5] Fix lint error --- lib/util/stream_utils.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index 7eb5f53744..c8211d0bb5 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -781,7 +781,8 @@ shaka.util.StreamUtils = class { } } - const keep = shaka.util.StreamUtils.supportedImageMimeTypes_.has(mimeType); + const keep = + shaka.util.StreamUtils.supportedImageMimeTypes_.has(mimeType); if (!keep) { shaka.log.debug('Dropping image stream. Is not supported by the ' + From 07f9bc466b320dc2bf9621e28b56b4e30d0345fa Mon Sep 17 00:00:00 2001 From: Alvaro Velad Date: Wed, 12 Jan 2022 18:25:38 +0100 Subject: [PATCH 4/5] Use a Map instead two Set --- lib/util/stream_utils.js | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index c8211d0bb5..36508264ca 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -765,24 +765,19 @@ shaka.util.StreamUtils = class { const imageStreams = []; for (const stream of manifest.imageStreams) { const mimeType = stream.mimeType; - if (!shaka.util.StreamUtils.supportedImageMimeTypes_.has(mimeType) && - !shaka.util.StreamUtils.unsupportedImageMimeTypes_.has(mimeType)) { + if (!shaka.util.StreamUtils.supportedImageMimeTypes_.has(mimeType)) { const minImage = shaka.util.StreamUtils.minImage.get(mimeType); if (minImage) { // eslint-disable-next-line no-await-in-loop const res = await shaka.util.StreamUtils.isImageSupported_(minImage); - if (res) { - shaka.util.StreamUtils.supportedImageMimeTypes_.add(mimeType); - } else { - shaka.util.StreamUtils.unsupportedImageMimeTypes_.add(mimeType); - } + shaka.util.StreamUtils.supportedImageMimeTypes_.set(mimeType, res); } else { - shaka.util.StreamUtils.unsupportedImageMimeTypes_.add(mimeType); + shaka.util.StreamUtils.supportedImageMimeTypes_.set(mimeType, false); } } const keep = - shaka.util.StreamUtils.supportedImageMimeTypes_.has(mimeType); + shaka.util.StreamUtils.supportedImageMimeTypes_.get(mimeType); if (!keep) { shaka.log.debug('Dropping image stream. Is not supported by the ' + @@ -1472,17 +1467,13 @@ shaka.util.StreamUtils.DecodingAttributes = { }; /** - * @private {!Set.} - */ -shaka.util.StreamUtils.supportedImageMimeTypes_ = new Set() - .add('image/svg+xml') - .add('image/png') - .add('image/jpeg'); - -/** - * @private {!Set.} + * @private {!Map.} */ -shaka.util.StreamUtils.unsupportedImageMimeTypes_ = new Set(); +shaka.util.StreamUtils.supportedImageMimeTypes_ = new Map() + .set('image/svg+xml', true) + .set('image/png', true) + .set('image/jpeg', true) + .set('image/jpg', true); /** * @const {string} From 0f64d54e661ad04d889d8b6de03dee8e9743c025 Mon Sep 17 00:00:00 2001 From: Alvaro Velad Date: Wed, 12 Jan 2022 19:34:53 +0100 Subject: [PATCH 5/5] Use underscore for private variables --- lib/util/stream_utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index 36508264ca..3ed813b9a6 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -766,7 +766,7 @@ shaka.util.StreamUtils = class { for (const stream of manifest.imageStreams) { const mimeType = stream.mimeType; if (!shaka.util.StreamUtils.supportedImageMimeTypes_.has(mimeType)) { - const minImage = shaka.util.StreamUtils.minImage.get(mimeType); + const minImage = shaka.util.StreamUtils.minImage_.get(mimeType); if (minImage) { // eslint-disable-next-line no-await-in-loop const res = await shaka.util.StreamUtils.isImageSupported_(minImage); @@ -1499,6 +1499,6 @@ shaka.util.StreamUtils.minAvifImage_ = 'data:image/avif;base64,AAAAIGZ0eXBhdm' + * @const {!Map.} * @private */ -shaka.util.StreamUtils.minImage = new Map() +shaka.util.StreamUtils.minImage_ = new Map() .set('image/webp', shaka.util.StreamUtils.minWebPImage_) .set('image/avif', shaka.util.StreamUtils.minAvifImage_);