enhanced essential property handling (#4385)

* adding essentialProps to settings

* add essentialProps to MediaInfo and Preferences

* Sschr/20240209 properties

* adding essentialProperties to MediaInfo-object

* resolving requested changes from review

* move thumbnail-schemeIds to Constants.js

src/core/Settings.js

@@ -72,6 +72,7 @@ import Events from './events/Events.js';
  *            enableManifestTimescaleMismatchFix: false,
  *            capabilities: {
  *               filterUnsupportedEssentialProperties: true,
+ *               supportedEssentialProperties: Constants.THUMBNAILS_SCHEME_ID_URIS,
  *               useMediaCapabilitiesApi: false
  *            },
  *            timeShiftBuffer: {
@@ -628,6 +629,8 @@ import Events from './events/Events.js';
  * @typedef {Object} Capabilities
  * @property {boolean} [filterUnsupportedEssentialProperties=true]
  * Enable to filter all the AdaptationSets and Representations which contain an unsupported \<EssentialProperty\> element.
+ * @property {Array.<string>} [supportedEssentialProperties=Constants.THUMBNAILS_SCHEME_ID_URIS]
+ * List of supported \<EssentialProperty\> elements
  * @property {boolean} [useMediaCapabilitiesApi=false]
  * Enable to use the MediaCapabilities API to check whether codecs are supported. If disabled MSE.isTypeSupported will be used instead.
  */
@@ -1005,6 +1008,7 @@ function Settings() {
             enableManifestTimescaleMismatchFix: false,
             capabilities: {
                 filterUnsupportedEssentialProperties: true,
+                supportedEssentialProperties: Constants.THUMBNAILS_SCHEME_ID_URIS,
                 useMediaCapabilitiesApi: false
             },
             timeShiftBuffer: {

src/dash/DashAdapter.js

@@ -992,6 +992,10 @@ function DashAdapter() {
         }
 
         mediaInfo.isText = dashManifestModel.getIsText(realAdaptation);
+        mediaInfo.essentialProperties = dashManifestModel.getEssentialPropertiesForAdaptation(realAdaptation);
+        if ((!mediaInfo.essentialProperties || mediaInfo.essentialProperties.length === 0) && realAdaptation.Representation && realAdaptation.Representation.length > 0) {
+            mediaInfo.essentialProperties = _getCommonRepresentationEssentialProperties(mediaInfo, realAdaptation);
+        }
         mediaInfo.supplementalProperties = dashManifestModel.getSupplementalPropertiesForAdaptation(realAdaptation);
         if ((!mediaInfo.supplementalProperties || mediaInfo.supplementalProperties.length === 0) && realAdaptation.Representation && realAdaptation.Representation.length > 0) {
             mediaInfo.supplementalProperties = _getCommonRepresentationSupplementalProperties(mediaInfo, realAdaptation);
@@ -1044,6 +1048,19 @@ function DashAdapter() {
         })
     }
 
+    function _getCommonRepresentationEssentialProperties(mediaInfo, realAdaptation) {
+        let arr = realAdaptation.Representation.map(repr => {
+            return dashManifestModel.getEssentialPropertiesForRepresentation(repr);
+        });
+
+        if (arr.every(v => JSON.stringify(v) === JSON.stringify(arr[0]))) {
+            // only output Representation.essentialProperties to mediaInfo, if they are present on all Representations
+            return arr[0];
+        }
+
+        return []
+    }
+
     function _getCommonRepresentationSupplementalProperties(mediaInfo, realAdaptation) {
         let arr = realAdaptation.Representation.map(repr => {
             return dashManifestModel.getSupplementalPropertiesForRepresentation(repr);

src/dash/models/DashManifestModel.js

@@ -47,7 +47,6 @@ import FactoryMaker from '../../core/FactoryMaker.js';
 import Debug from '../../core/Debug.js';
 import DashJSError from '../../streaming/vo/DashJSError.js';
 import Errors from '../../core/errors/Errors.js';
-import {THUMBNAILS_SCHEME_ID_URIS} from '../../streaming/thumbnail/ThumbnailTracks.js';
 import MpdLocation from '../vo/MpdLocation.js';
 import PatchLocation from '../vo/PatchLocation.js';
 import ContentProtection from '../vo/ContentProtection.js';
@@ -83,7 +82,7 @@ function DashManifestModel() {
         // Check for thumbnail images
         if (adaptation.Representation && adaptation.Representation.length) {
             const essentialProperties = getEssentialPropertiesForRepresentation(adaptation.Representation[0]);
-            if (essentialProperties && essentialProperties.length > 0 && THUMBNAILS_SCHEME_ID_URIS.indexOf(essentialProperties[0].schemeIdUri) >= 0) {
+            if (essentialProperties && essentialProperties.length > 0 && Constants.THUMBNAILS_SCHEME_ID_URIS.indexOf(essentialProperties[0].schemeIdUri) >= 0) {
                 return (type === Constants.IMAGE);
             }
         }
@@ -566,14 +565,22 @@ function DashManifestModel() {
         }
     }
 
+    function getEssentialPropertiesForAdaptation(adaptation) {
+        if (!adaptation || !adaptation.hasOwnProperty(DashConstants.ESSENTIAL_PROPERTY) || !adaptation.EssentialProperty.length) return [];
+        return adaptation.EssentialProperty.map( essentialProperty => {
+            const s = new DescriptorType();
+            s.init(essentialProperty);
+            return s
+        });
+    }
+
     function getEssentialPropertiesForRepresentation(realRepresentation) {
-        if (!realRepresentation || !realRepresentation.EssentialProperty || !realRepresentation.EssentialProperty.length) return null;
+        if (!realRepresentation || !realRepresentation.EssentialProperty || !realRepresentation.EssentialProperty.length) return [];
 
-        return realRepresentation.EssentialProperty.map((prop) => {
-            return {
-                schemeIdUri: prop.schemeIdUri,
-                value: prop.value
-            };
+        return realRepresentation.EssentialProperty.map((essentialProperty) => {
+            const s = new DescriptorType();
+            s.init(essentialProperty);
+            return s
         });
     }
 
@@ -1399,6 +1406,7 @@ function DashManifestModel() {
         getContentProtectionByPeriod,
         getContentSteering,
         getDuration,
+        getEssentialPropertiesForAdaptation,
         getEssentialPropertiesForRepresentation,
         getEventStreamForAdaptationSet,
         getEventStreamForRepresentation,

src/dash/vo/MediaInfo.js

@@ -55,7 +55,8 @@ class MediaInfo {
         this.selectionPriority = 1;
         this.streamInfo = null;
         this.subSegmentAlignment = false;
-        this.supplementalProperties = {};
+        this.supplementalProperties = [];
+        this.essentialProperties = [];
         this.type = null;
         this.viewpoint = null;
     }

src/streaming/constants/Constants.js

@@ -225,6 +225,7 @@ export default {
     START_TIME: 'starttime',
     SERVICE_DESCRIPTION_DVB_LL_SCHEME: 'urn:dvb:dash:lowlatency:scope:2019',
     SUPPLEMENTAL_PROPERTY_DVB_LL_SCHEME: 'urn:dvb:dash:lowlatency:critical:2019',
+    THUMBNAILS_SCHEME_ID_URIS: ['http://dashif.org/thumbnail_tile', 'http://dashif.org/guidelines/thumbnail_tile'],
     XML: 'XML',
     ARRAY_BUFFER: 'ArrayBuffer',
     DVB_REPORTING_URL: 'dvb:reportingUrl',

src/streaming/thumbnail/ThumbnailTracks.js

@@ -39,9 +39,6 @@ import XHRLoader from '../../streaming/net/XHRLoader.js';
 import DashHandler from '../../dash/DashHandler.js';
 import SegmentsController from '../../dash/controllers/SegmentsController.js';
 
-export const THUMBNAILS_SCHEME_ID_URIS = ['http://dashif.org/thumbnail_tile',
-    'http://dashif.org/guidelines/thumbnail_tile'];
-
 function ThumbnailTracks(config) {
     const context = this.context;
     const adapter = config.adapter;
@@ -142,7 +139,7 @@ function ThumbnailTracks(config) {
 
         if (representation.essentialProperties) {
             representation.essentialProperties.forEach((p) => {
-                if (THUMBNAILS_SCHEME_ID_URIS.indexOf(p.schemeIdUri) >= 0 && p.value) {
+                if (Constants.THUMBNAILS_SCHEME_ID_URIS.indexOf(p.schemeIdUri) >= 0 && p.value) {
                     const vars = p.value.split('x');
                     if (vars.length === 2 && !isNaN(vars[0]) && !isNaN(vars[1])) {
                         track.tilesHor = parseInt(vars[0], 10);

src/streaming/utils/Capabilities.js

@@ -29,7 +29,6 @@
  *  POSSIBILITY OF SUCH DAMAGE.
  */
 import FactoryMaker from '../../core/FactoryMaker.js';
-import {THUMBNAILS_SCHEME_ID_URIS} from '../thumbnail/ThumbnailTracks.js';
 import Constants from '../constants/Constants.js';
 
 export function supportsMediaSource() {
@@ -187,8 +186,9 @@ function Capabilities() {
      * @return {boolean}
      */
     function supportsEssentialProperty(ep) {
+        let supportedEssentialProps = settings.get().streaming.capabilities.supportedEssentialProperties;
         try {
-            return THUMBNAILS_SCHEME_ID_URIS.indexOf(ep.schemeIdUri) !== -1;
+            return supportedEssentialProps.indexOf(ep.schemeIdUri) !== -1;
         } catch (e) {
             return true;
         }

test/unit/dash.DashAdapter.js

@@ -59,11 +59,24 @@ const manifest_with_ll_service_description = {
         }]
     }]
 };
-const manifest_without_supplemental_properties = {
+const manifest_without_properties = {
     loadedTime: new Date(),
     mediaPresentationDuration: 10,
     Period: [{ AdaptationSet: [{ id: 0, mimeType: Constants.VIDEO }] }]
 };
+const manifest_with_essential_properties = {
+    loadedTime: new Date(),
+    mediaPresentationDuration: 10,
+    Period: [{
+        AdaptationSet: [{
+            id: 0, mimeType: Constants.VIDEO,
+            EssentialProperty: [{
+                schemeIdUri: 'test:scheme:essp',
+                value: 'value1'
+            }, { schemeIdUri: 'test:scheme:essp', value: 'value2' }]
+        }]
+    }]
+};
 const manifest_with_supplemental_properties = {
     loadedTime: new Date(),
     mediaPresentationDuration: 10,
@@ -77,6 +90,34 @@ const manifest_with_supplemental_properties = {
         }]
     }]
 };
+const manifest_with_essential_properties_on_repr = {
+    loadedTime: new Date(),
+    mediaPresentationDuration: 10,
+    Period: [{
+        AdaptationSet: [{
+            id: 0, mimeType: Constants.VIDEO,
+            // SupplementalProperty: [{schemeIdUri: 'test:scheme', value: 'value1'},{schemeIdUri: 'test:scheme', value: 'value2'},{schemeIdUri: 'test:scheme', value: 'value3'}],
+            [DashConstants.REPRESENTATION]: [
+                {
+                    id: 10, bandwidth: 128000,
+                    [DashConstants.ESSENTIAL_PROPERTY]: [
+                        { schemeIdUri: 'test:scheme', value: 'value1' },
+                        { schemeIdUri: 'test:scheme', value: 'value2' },
+                        { schemeIdUri: 'test:scheme', value: 'value3' }
+                    ]
+                },
+                {
+                    id: 11, bandwidth: 160000,
+                    [DashConstants.ESSENTIAL_PROPERTY]: [
+                        { schemeIdUri: 'test:scheme', value: 'value1' },
+                        { schemeIdUri: 'test:scheme', value: 'value2' },
+                        { schemeIdUri: 'test:scheme', value: 'value3' }
+                    ]
+                }
+            ]
+        }]
+    }]
+};
 const manifest_with_supplemental_properties_on_repr = {
     loadedTime: new Date(),
     mediaPresentationDuration: 10,
@@ -105,6 +146,30 @@ const manifest_with_supplemental_properties_on_repr = {
         }]
     }]
 };
+const manifest_with_essential_properties_on_only_one_repr = {
+    loadedTime: new Date(),
+    mediaPresentationDuration: 10,
+    Period: [{
+        AdaptationSet: [{
+            id: 0, mimeType: Constants.VIDEO,
+            [DashConstants.REPRESENTATION]: [
+                {
+                    id: 10, bandwidth: 128000,
+                    [DashConstants.ESSENTIAL_PROPERTY]: [
+                        { schemeIdUri: 'test:scheme', value: 'value1' },
+                        { schemeIdUri: 'test:scheme', value: 'value2' }
+                    ]
+                },
+                {
+                    id: 11, bandwidth: 160000
+                },
+                {
+                    id: 12, bandwidth: 96000
+                }
+            ]
+        }]
+    }]
+};
 const manifest_with_supplemental_properties_on_only_one_repr = {
     loadedTime: new Date(),
     mediaPresentationDuration: 10,
@@ -559,11 +624,73 @@ describe('DashAdapter', function () {
             });
 
             describe('mediainfo populated from manifest', function () {
+                it('essential properties should be empty if not defined', function () {
+                    const mediaInfoArray = dashAdapter.getAllMediaInfoForType({
+                        id: 'defaultId_0',
+                        index: 0
+                    }, Constants.VIDEO, manifest_without_properties);
+
+                    expect(mediaInfoArray).to.be.instanceOf(Array);
+                    expect(mediaInfoArray.length).equals(1);
+
+                    expect(mediaInfoArray[0].essentialProperties).to.be.instanceOf(Array);
+                    expect(mediaInfoArray[0].essentialProperties.length).equals(0);
+                });
+
+                it('essential properties should be filled if correctly defined', function () {
+                    const mediaInfoArray = dashAdapter.getAllMediaInfoForType({
+                        id: 'defaultId_0',
+                        index: 0
+                    }, Constants.VIDEO, manifest_with_essential_properties);
+
+                    expect(mediaInfoArray).to.be.instanceOf(Array);
+                    expect(mediaInfoArray.length).equals(1);
+
+                    expect(mediaInfoArray[0].codec).to.be.null;
+
+                    expect(mediaInfoArray[0].essentialProperties).to.be.instanceOf(Array);
+                    expect(mediaInfoArray[0].essentialProperties.length).equals(2);
+                });
+
+                it('essential properties should be filled if set on all representations', function () {
+                    const mediaInfoArray = dashAdapter.getAllMediaInfoForType({
+                        id: 'defaultId_0',
+                        index: 0
+                    }, Constants.VIDEO, manifest_with_essential_properties_on_repr);
+
+                    expect(mediaInfoArray).to.be.instanceOf(Array);
+                    expect(mediaInfoArray.length).equals(1);
+
+                    expect(mediaInfoArray[0].representationCount).equals(2);
+                    expect(mediaInfoArray[0].codec).not.to.be.null;
+
+                    expect(mediaInfoArray[0].essentialProperties).to.be.instanceOf(Array);
+                    expect(mediaInfoArray[0].essentialProperties.length).equals(3);
+
+                    expect(mediaInfoArray[0].essentialProperties[1].schemeIdUri).equals('test:scheme');
+                    expect(mediaInfoArray[0].essentialProperties[1].value).equals('value2');
+                });
+
+                it('essential properties should not be filled if not set on all representations', function () {
+                    const mediaInfoArray = dashAdapter.getAllMediaInfoForType({
+                        id: 'defaultId_0',
+                        index: 0
+                    }, Constants.VIDEO, manifest_with_essential_properties_on_only_one_repr);
+
+                    expect(mediaInfoArray).to.be.instanceOf(Array);
+                    expect(mediaInfoArray.length).equals(1);
+
+                    expect(mediaInfoArray[0].representationCount).equals(3);
+
+                    expect(mediaInfoArray[0].essentialProperties).to.be.instanceOf(Array);
+                    expect(mediaInfoArray[0].essentialProperties.length).equals(0);
+                });
+
                 it('supplemental properties should be empty if not defined', function () {
                     const mediaInfoArray = dashAdapter.getAllMediaInfoForType({
                         id: 'defaultId_0',
                         index: 0
-                    }, Constants.VIDEO, manifest_without_supplemental_properties);
+                    }, Constants.VIDEO, manifest_without_properties);
 
                     expect(mediaInfoArray).to.be.instanceOf(Array);
                     expect(mediaInfoArray.length).equals(1);

test/unit/dash.models.DashManifestModel.js

@@ -130,6 +130,62 @@ describe('DashManifestModel', function () {
             expect(rolesArray).to.be.empty;
         });
 
+        it('should return an empty array when getEssentialPropertiesForAdaptation', () => {
+            const suppPropArray = dashManifestModel.getEssentialPropertiesForAdaptation();
+
+            expect(suppPropArray).to.be.instanceOf(Object);
+            expect(suppPropArray).to.be.empty;
+        });
+
+        it('should return an empty array when getEssentialPropertiesForAdaptation', () => {
+            const suppPropArray = dashManifestModel.getEssentialPropertiesForAdaptation();
+
+            expect(suppPropArray).to.be.instanceOf(Array);
+            expect(suppPropArray).to.be.empty;
+        });
+
+        it('should return correct array of DescriptorType when getEssentialPropertiesForAdaptation is called', () => {
+            const essPropArray = dashManifestModel.getEssentialPropertiesForAdaptation({
+                EssentialProperty: [{ schemeIdUri: 'test.scheme', value: 'testVal' }, {
+                    schemeIdUri: 'test.scheme',
+                    value: 'test2Val'
+                }]
+            });
+
+            expect(essPropArray).to.be.instanceOf(Array);
+            expect(essPropArray.length).to.equal(2);
+            expect(essPropArray[0]).to.be.instanceOf(DescriptorType);
+            expect(essPropArray[0].schemeIdUri).equals('test.scheme');
+            expect(essPropArray[0].value).equals('testVal');
+            expect(essPropArray[1].schemeIdUri).equals('test.scheme');
+            expect(essPropArray[1].value).equals('test2Val');
+        });
+
+        it('should return an empty array when getEssentialPropertiesForRepresentation', () => {
+            const essPropArray = dashManifestModel.getEssentialPropertiesForRepresentation();
+
+            expect(essPropArray).to.be.instanceOf(Object);
+            expect(essPropArray).to.be.empty;
+        });
+
+        it('should return an empty array when getEssentialPropertiesForRepresentation', () => {
+            const essPropArray = dashManifestModel.getEssentialPropertiesForRepresentation();
+
+            expect(essPropArray).to.be.instanceOf(Array);
+            expect(essPropArray).to.be.empty;
+        });
+
+        it('should return correct array of DescriptorType when getEssentialPropertiesForRepresentation is called', () => {
+            const essPropArray = dashManifestModel.getEssentialPropertiesForRepresentation({
+                EssentialProperty: [{ schemeIdUri: 'test.scheme', value: 'testVal' }]
+            });
+
+            expect(essPropArray).to.be.instanceOf(Array);
+            expect(essPropArray[0]).to.be.instanceOf(DescriptorType);
+            expect(essPropArray[0].schemeIdUri).equals('test.scheme');
+            expect(essPropArray[0].value).equals('testVal');
+        });
+
         it('should return an empty array when getSupplementalPropertiesForAdaptation', () => {
             const suppPropArray = dashManifestModel.getSupplementalPropertiesForAdaptation();