feat: add option for segment-relative VTT timings (#4083)

This PR fixes #3242 where for some live streams using segmented VTT, text timings are relative to segment start instead of being absolute.

The PR introduces a new setting: `manifest.segmentRelativeVttTiming: boolean` allowing such alternative timing offset calculation.

The setting is off by default, preserving the current player behaviour.

Co-authored-by: Joey Parrish <joeyparrish@users.noreply.github.com>

demo/common/message_ids.js

@@ -232,6 +232,7 @@ shakaDemo.MessageIds = {
   RESTRICTIONS_SECTION_HEADER: 'DEMO_RESTRICTIONS_SECTION_HEADER',
   SAFE_SEEK_OFFSET: 'DEMO_SAFE_SEEK_OFFSET',
   SAFE_SKIP_DISTANCE: 'DEMO_SAFE_SKIP_DISTANCE',
+  SEGMENT_RELATIVE_VTT_TIMING: 'DEMO_SEGMENT_RELATIVE_VTT_TIMING',
   SESSION_ID: 'DEMO_SESSION_ID',
   SHAKA_CONTROLS: 'DEMO_SHAKA_CONTROLS',
   SLOW_HALF_LIFE: 'DEMO_SLOW_HALF_LIFE',

demo/config.js

@@ -237,7 +237,9 @@ shakaDemo.Config = class {
         .addBoolInput_(MessageIds.DISABLE_TEXT,
             'manifest.disableText')
         .addBoolInput_(MessageIds.DISABLE_THUMBNAILS,
-            'manifest.disableThumbnails');
+            'manifest.disableThumbnails')
+        .addBoolInput_(MessageIds.SEGMENT_RELATIVE_VTT_TIMING,
+            'manifest.segmentRelativeVttTiming');
 
     this.addRetrySection_('manifest', MessageIds.MANIFEST_RETRY_SECTION_HEADER);
   }

demo/locales/en.json

@@ -180,6 +180,7 @@
   "DEMO_RESTRICTIONS_SECTION_HEADER": "Restrictions",
   "DEMO_SAFE_SEEK_OFFSET": "Safe Seek Offset",
   "DEMO_SAFE_SKIP_DISTANCE": "Safe Skip Distance",
+  "DEMO_SEGMENT_RELATIVE_VTT_TIMING": "Enable segment-relative VTT Timing",
   "DEMO_SESSION_ID": "Session ID",
   "DEMO_SAVE_BUTTON": "Save",
   "DEMO_SHAKA": "Shaka",

demo/locales/source.json

@@ -727,6 +727,10 @@
     "description": "A button to save a custom asset.",
     "message": "Save"
   },
+  "DEMO_SEGMENT_RELATIVE_VTT_TIMING": {
+    "description": "The name of a configuration value.",
+    "message": "Enable segment-relative VTT Timing"
+  },
   "DEMO_SESSION_ID": {
     "description": "The name of a configuration value.",
     "message": "Session ID"

externs/shaka/player.js

@@ -792,6 +792,7 @@ shaka.extern.HlsManifestConfiguration;
  *   disableText: boolean,
  *   disableThumbnails: boolean,
  *   defaultPresentationDelay: number,
+ *   segmentRelativeVttTiming: boolean,
  *   dash: shaka.extern.DashManifestConfiguration,
  *   hls: shaka.extern.HlsManifestConfiguration
  * }}
@@ -823,6 +824,10 @@ shaka.extern.HlsManifestConfiguration;
  *   configured or set as 0.
  *   For HLS, the default value is 3 segments duration if not configured or
  *   set as 0.
+ * @property {boolean} segmentRelativeVttTiming
+ *   Option to calculate VTT text timings relative to the segment start
+ *   instead of relative to the period start (which is the default).
+ *   Defaults to <code>false</code>.
  * @property {shaka.extern.DashManifestConfiguration} dash
  *   Advanced parameters used by the DASH manifest parser.
  * @property {shaka.extern.HlsManifestConfiguration} hls

externs/shaka/text.js

@@ -442,7 +442,8 @@ shaka.extern.TextParser = class {
  * @typedef {{
  *   periodStart: number,
  *   segmentStart: number,
- *   segmentEnd: number
+ *   segmentEnd: number,
+ *   vttOffset: number
  * }}
  *
  * @property {number} periodStart
@@ -451,6 +452,9 @@ shaka.extern.TextParser = class {
  *     The absolute start time of the segment in seconds.
  * @property {number} segmentEnd
  *     The absolute end time of the segment in seconds.
+ * @property {number} vttOffset
+ *     The start time relative to either segment or period start depending
+ *     on <code>segmentRelativeVttTiming</code> configuration.
  *
  * @exportDoc
  */

lib/media/media_source_engine.js

@@ -66,6 +66,9 @@ shaka.media.MediaSourceEngine = class {
     /** @private {shaka.text.TextEngine} */
     this.textEngine_ = null;
 
+    /** @private {boolean} */
+    this.segmentRelativeVttTiming_ = false;
+
     const onMetadataNoOp = (metadata, timestampOffset, segmentEnd) => {};
 
     /** @private {!function(!Array.<shaka.extern.ID3Metadata>,
@@ -367,7 +370,8 @@ shaka.media.MediaSourceEngine = class {
     if (!this.textEngine_) {
       this.textEngine_ = new shaka.text.TextEngine(this.textDisplayer_);
     }
-    this.textEngine_.initParser(mimeType, sequenceMode);
+    this.textEngine_.initParser(mimeType, sequenceMode,
+        this.segmentRelativeVttTiming_);
   }
 
   /**
@@ -1113,6 +1117,13 @@ shaka.media.MediaSourceEngine = class {
     }
   }
 
+  /**
+   * @param {boolean} segmentRelativeVttTiming
+   */
+  setSegmentRelativeVttTiming(segmentRelativeVttTiming) {
+    this.segmentRelativeVttTiming_ = segmentRelativeVttTiming;
+  }
+
   /**
    * Apply platform-specific transformations to this segment to work around
    * issues in the platform.

lib/player.js

@@ -1571,6 +1571,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
         (metadata, offset, endTime) => {
           this.processTimedMetadataMediaSrc_(metadata, offset, endTime);
         });
+    const {segmentRelativeVttTiming} = this.config_.manifest;
+    mediaSourceEngine.setSegmentRelativeVttTiming(segmentRelativeVttTiming);
 
     // Wait for media source engine to finish opening. This promise should
     // NEVER be rejected as per the media source engine implementation.
@@ -3020,6 +3022,10 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     }
 
     if (this.mediaSourceEngine_) {
+      const {segmentRelativeVttTiming} = this.config_.manifest;
+      this.mediaSourceEngine_.setSegmentRelativeVttTiming(
+          segmentRelativeVttTiming);
+
       const textDisplayerFactory = this.config_.textDisplayFactory;
       if (this.lastTextFactory_ != textDisplayerFactory) {
         const displayer =
@@ -4864,6 +4870,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
         periodStart: 0,
         segmentStart: 0,
         segmentEnd: this.video_.duration,
+        vttOffset: 0,
       };
       const data = shaka.util.BufferUtils.toUint8(buffer);
       const cues = obj.parseMedia(data, time);

lib/text/text_engine.js

@@ -31,6 +31,9 @@ shaka.text.TextEngine = class {
     /** @private {shaka.extern.TextDisplayer} */
     this.displayer_ = displayer;
 
+    /** @private {boolean} */
+    this.segmentRelativeVttTiming_ = false;
+
     /** @private {number} */
     this.timestampOffset_ = 0;
 
@@ -130,8 +133,9 @@ shaka.text.TextEngine = class {
    *
    * @param {string} mimeType
    * @param {boolean} sequenceMode
+   * @param {boolean} segmentRelativeVttTiming
    */
-  initParser(mimeType, sequenceMode) {
+  initParser(mimeType, sequenceMode, segmentRelativeVttTiming) {
     // No parser for CEA, which is extracted from video and side-loaded
     // into TextEngine and TextDisplayer.
     if (mimeType == shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE ||
@@ -149,6 +153,7 @@ shaka.text.TextEngine = class {
       shaka.log.alwaysWarn(
           'Text parsers should have a "setSequenceMode" method!');
     }
+    this.segmentRelativeVttTiming_ = segmentRelativeVttTiming;
   }
 
   /**
@@ -174,11 +179,15 @@ shaka.text.TextEngine = class {
       return;
     }
 
+    const vttOffset = this.segmentRelativeVttTiming_ ?
+        startTime : this.timestampOffset_;
+
     /** @type {shaka.extern.TextParser.TimeContext} **/
     const time = {
       periodStart: this.timestampOffset_,
       segmentStart: startTime,
       segmentEnd: endTime,
+      vttOffset: vttOffset,
     };
 
     // Parse the buffer and add the new cues.

lib/text/vtt_text_parser.js

@@ -62,9 +62,13 @@ shaka.text.VttTextParser = class {
           shaka.util.Error.Code.INVALID_TEXT_HEADER);
     }
 
+    // Depending on "segmentRelativeVttTiming" configuration,
+    // "vttOffset" will correspond to either "periodStart" (default)
+    // or "segmentStart", for segmented VTT where timings are relative
+    // to the beginning of each segment.
     // NOTE: "periodStart" is the timestamp offset applied via TextEngine.
     // It is no longer closely tied to periods, but the name stuck around.
-    let offset = time.periodStart;
+    let offset = time.vttOffset;
 
     // Do not honor the 'X-TIMESTAMP-MAP' value when in sequence mode.
     // That is because it is used mainly (solely?) to account for the timestamp

lib/util/player_configuration.js

@@ -89,6 +89,7 @@ shaka.util.PlayerConfiguration = class {
       disableText: false,
       disableThumbnails: false,
       defaultPresentationDelay: 0,
+      segmentRelativeVttTiming: false,
       dash: {
         clockSyncUri: '',
         ignoreDrmInfo: false,

test/player_unit.js

@@ -140,6 +140,8 @@ describe('Player', () => {
             jasmine.createSpy('destroy').and.returnValue(Promise.resolve()),
         setUseEmbeddedText: jasmine.createSpy('setUseEmbeddedText'),
         getUseEmbeddedText: jasmine.createSpy('getUseEmbeddedText'),
+        setSegmentRelativeVttTiming:
+            jasmine.createSpy('setSegmentRelativeVttTiming'),
         getTextDisplayer: () => textDisplayer,
         ended: jasmine.createSpy('ended').and.returnValue(false),
       };

test/test/util/fake_media_source_engine.js

@@ -132,6 +132,10 @@ shaka.test.FakeMediaSourceEngine = class {
     this.getTextDisplayer =
         jasmine.createSpy('getTextDisplayer')
             .and.returnValue(new shaka.test.FakeTextDisplayer());
+
+    /** @type {!jasmine.Spy} */
+    this.setSegmentRelativeVttTiming =
+        jasmine.createSpy('setSegmentRelativeVttTiming').and.stub();
   }
 
   /** @override */

test/text/cue_integration.js

@@ -23,7 +23,7 @@ describe('Cue', () => {
         'WEBVTT\n\n' +
         '00:00:20.000 --> 00:00:40.000\n' +
         'Test',
-        {periodStart: 7, segmentStart: 10, segmentEnd: 60});
+        {periodStart: 7, segmentStart: 10, segmentEnd: 60, vttOffset: 7});
     expect(cues.length).toBe(1);
     expect(cues[0].startTime).toBe(27);
     expect(cues[0].endTime).toBe(47);
@@ -38,7 +38,7 @@ describe('Cue', () => {
         'ID1\n' +
         '00:00:20.000 --> 00:00:40.000 align:middle size:56% vertical:lr\n' +
         'Test',
-        {periodStart: 0, segmentStart: 0, segmentEnd: 0});
+        {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0});
     expect(cues.length).toBe(1);
   });
 

test/text/lrc_text_parser_unit.js

@@ -12,7 +12,7 @@ describe('LrcTextParser', () => {
   it('supports no cues', () => {
     verifyHelper([],
         '',
-        {periodStart: 0, segmentStart: 0, segmentEnd: 0});
+        {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0});
   });
 
   it('handles a blank line at the start of the file', () => {
@@ -22,7 +22,7 @@ describe('LrcTextParser', () => {
         ],
         '\n\n' +
         '[00:00.00]Test',
-        {periodStart: 0, segmentStart: 0, segmentEnd: 0});
+        {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0});
   });
 
   it('handles a blank line at the end of the file', () => {
@@ -32,7 +32,7 @@ describe('LrcTextParser', () => {
         ],
         '[00:00.00]Test' +
         '\n\n',
-        {periodStart: 0, segmentStart: 0, segmentEnd: 0});
+        {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0});
   });
 
   it('handles no blank line at the end of the file', () => {
@@ -41,8 +41,7 @@ describe('LrcTextParser', () => {
           {startTime: 0, endTime: 2, payload: 'Test'},
         ],
         '[00:00.00]Test',
-        {periodStart: 0, segmentStart: 0, segmentEnd: 0,
-        });
+        {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0});
   });
 
   it('supports multiple cues', () => {
@@ -55,7 +54,7 @@ describe('LrcTextParser', () => {
         '[00:00.00]Test\n' +
         '[00:10.00]Test2\n' +
         '[00:20.00]Test3',
-        {periodStart: 0, segmentStart: 0, segmentEnd: 0});
+        {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0});
   });
 
   it('supports different time formats', () => {
@@ -74,7 +73,7 @@ describe('LrcTextParser', () => {
         '[00:30,1]Test4\n' +
         '[00:40,001]Test5\n' +
         '[00:50,02]Test6',
-        {periodStart: 0, segmentStart: 0, segmentEnd: 0});
+        {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0});
   });
 
   /**

test/text/mp4_ttml_parser_unit.js

@@ -47,7 +47,7 @@ describe('Mp4TtmlParser', () => {
   it('handles media segments with multiple mdats', () => {
     const parser = new shaka.text.Mp4TtmlParser();
     parser.parseInit(ttmlInitSegment);
-    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0};
+    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0};
     const ret = parser.parseMedia(ttmlSegmentMultipleMDAT, time);
     // Bodies.
     expect(ret.length).toBe(2);
@@ -60,8 +60,10 @@ describe('Mp4TtmlParser', () => {
   });
 
   it('accounts for offset', () => {
-    const time1 = {periodStart: 0, segmentStart: 0, segmentEnd: 0};
-    const time2 = {periodStart: 7, segmentStart: 0, segmentEnd: 0};
+    const time1 =
+        {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0};
+    const time2 =
+        {periodStart: 7, segmentStart: 0, segmentEnd: 0, vttOffset: 7};
 
     const parser = new shaka.text.Mp4TtmlParser();
     parser.parseInit(ttmlInitSegment);
@@ -165,7 +167,7 @@ describe('Mp4TtmlParser', () => {
     ];
     const parser = new shaka.text.Mp4TtmlParser();
     parser.parseInit(ttmlInitSegment);
-    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0};
+    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0};
     const result = parser.parseMedia(ttmlSegment, time);
     shaka.test.TtmlUtils.verifyHelper(
         cues, result, {startTime: 23, endTime: 53.5});

test/text/mp4_vtt_parser_unit.js

@@ -71,7 +71,7 @@ describe('Mp4VttParser', () => {
 
     const parser = new shaka.text.Mp4VttParser();
     parser.parseInit(vttInitSegment);
-    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0};
+    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0};
     const result = parser.parseMedia(vttSegment, time);
     verifyHelper(cues, result);
   });
@@ -99,7 +99,7 @@ describe('Mp4VttParser', () => {
 
     const parser = new shaka.text.Mp4VttParser();
     parser.parseInit(vttInitSegment);
-    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0};
+    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0};
     const result = parser.parseMedia(vttSegmentMultiPayload, time);
     verifyHelper(cues, result);
   });
@@ -127,7 +127,7 @@ describe('Mp4VttParser', () => {
 
     const parser = new shaka.text.Mp4VttParser();
     parser.parseInit(vttInitSegment);
-    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0};
+    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0};
     const result = parser.parseMedia(vttSegSettings, time);
     verifyHelper(cues, result);
   });
@@ -149,7 +149,7 @@ describe('Mp4VttParser', () => {
 
     const parser = new shaka.text.Mp4VttParser();
     parser.parseInit(vttInitSegment);
-    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0};
+    const time = {periodStart: 0, segmentStart: 0, segmentEnd: 0, vttOffset: 0};
     const result = parser.parseMedia(vttSegNoDuration, time);
     verifyHelper(cues, result);
   });
@@ -171,7 +171,8 @@ describe('Mp4VttParser', () => {
 
     const parser = new shaka.text.Mp4VttParser();
     parser.parseInit(vttInitSegment);
-    const time = {periodStart: 10, segmentStart: 0, segmentEnd: 0};
+    const time =
+        {periodStart: 10, segmentStart: 0, segmentEnd: 0, vttOffset: 10};
     const result = parser.parseMedia(vttSegment, time);
     verifyHelper(cues, result);
   });