feat(hls): Read EXT-X-PROGRAM-DATE-TIME

This makes the HLS parser read the EXT-X-PROGRAM-DATE-TIME value
on manifests, and use it to make sure that segments are inserted at
the correct place in the timeline, when in sequence mode.

demo/common/message_ids.js

@@ -188,6 +188,7 @@ shakaDemo.MessageIds = {
   IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY: 'DEMO_IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY',
   IGNORE_HLS_IMAGE_FAILURES: 'DEMO_IGNORE_HLS_IMAGE_FAILURES',
   IGNORE_HLS_TEXT_FAILURES: 'DEMO_IGNORE_HLS_TEXT_FAILURES',
+  IGNORE_MANIFEST_PROGRAM_DATE_TIME: 'DEMO_IGNORE_MANIFEST_PROGRAM_DATE_TIME',
   IGNORE_MIN_BUFFER_TIME: 'DEMO_IGNORE_MIN_BUFFER_TIME',
   IGNORE_TEXT_FAILURES: 'DEMO_IGNORE_TEXT_FAILURES',
   INACCURATE_MANIFEST_TOLERANCE: 'DEMO_INACCURATE_MANIFEST_TOLERANCE',

demo/config.js

@@ -213,6 +213,8 @@ shakaDemo.Config = class {
             'manifest.hls.defaultAudioCodec')
         .addTextInput_(MessageIds.DEFAULT_VIDEO_CODEC,
             'manifest.hls.defaultVideoCodec')
+        .addBoolInput_(MessageIds.IGNORE_MANIFEST_PROGRAM_DATE_TIME,
+            'manifest.hls.ignoreManifestProgramDateTime')
         .addNumberInput_(MessageIds.AVAILABILITY_WINDOW_OVERRIDE,
             'manifest.availabilityWindowOverride',
             /* canBeDecimal= */ true,

demo/locales/en.json

@@ -99,6 +99,7 @@
   "DEMO_IMA_ASSET_KEY": "Asset key (for LIVE DAI Content)",
   "DEMO_IMA_CONTENT_SRC_ID": "Content source ID (for VOD DAI Content)",
   "DEMO_IMA_VIDEO_ID": "Video ID (for VOD DAI Content)",
+  "DEMO_IGNORE_MANIFEST_PROGRAM_DATE_TIME": "Ignore Program Date Time from manifest",
   "DEMO_IGNORE_MIN_BUFFER_TIME": "Ignore Min Buffer Time",
   "DEMO_IGNORE_TEXT_FAILURES": "Ignore Text Stream Failures",
   "DEMO_INACCURATE_MANIFEST_TOLERANCE": "Inaccurate Manifest Tolerance",

demo/locales/source.json

@@ -399,6 +399,10 @@
     "description": "The label on a field that allows users to provide a video id for a custom asset.",
     "message": "Video ID (for VOD DAI Content)"
   },
+  "DEMO_IGNORE_MANIFEST_PROGRAM_DATE_TIME": {
+    "description": "The name of a configuration value.",
+    "message": "Ignore Program Date Time from manifest"
+  },
   "DEMO_IGNORE_MIN_BUFFER_TIME": {
     "description": "The name of a configuration value.",
     "message": "Ignore Min Buffer Time"

externs/shaka/player.js

@@ -747,7 +747,8 @@ shaka.extern.DashManifestConfiguration;
  *   ignoreTextStreamFailures: boolean,
  *   ignoreImageStreamFailures: boolean,
  *   defaultAudioCodec: string,
- *   defaultVideoCodec: string
+ *   defaultVideoCodec: string,
+ *   ignoreManifestProgramDateTime: boolean
  * }}
  *
  * @property {boolean} ignoreTextStreamFailures
@@ -762,6 +763,13 @@ shaka.extern.DashManifestConfiguration;
  * @property {string} defaultVideoCodec
  *   The default video codec if it is not specified in the HLS playlist.
  *   <i>Defaults to <code>'avc1.42E01E'</code>.</i>
+ * @property {boolean} ignoreManifestProgramDateTime
+ *   If <code>true</code>, the HLS parser will ignore the
+ *   <code>EXT-X-PROGRAM-DATE-TIME</code> tags in the manifest, and always
+ *   create synthetic sync times.
+ *   Meant for content that should be synchronized, but has broken values for
+ *   synchronization.
+ *   <i>Defaults to <code>false</code>.</i>
  * @exportDoc
  */
 shaka.extern.HlsManifestConfiguration;

lib/hls/hls_parser.js

@@ -38,6 +38,7 @@ goog.require('shaka.util.OperationManager');
 goog.require('shaka.util.Pssh');
 goog.require('shaka.util.Timer');
 goog.require('shaka.util.Platform');
+goog.require('shaka.util.XmlUtils');
 goog.requireType('shaka.hls.Segment');
 
 
@@ -120,6 +121,16 @@ shaka.hls.HlsParser = class {
      */
     this.updatePlaylistDelay_ = 0;
 
+    /**
+     * A time offset to apply to EXT-X-PROGRAM-DATE-TIME values to normalize
+     * them so that they start at 0, and are in scale with the normal time
+     * values.
+     * Starts as null, which represents how we cannot know how to properly
+     * normalize the values until we have seen each playlist.
+     * @private {?number}
+     */
+    this.syncTimeOffset_ = null;
+
     /**
      * This timer is used to trigger the start of a manifest update. A manifest
      * update is async. Once the update is finished, the timer will be restarted
@@ -314,7 +325,7 @@ shaka.hls.HlsParser = class {
     const segments = this.createSegments_(
         streamInfo.verbatimMediaPlaylistUri, playlist, stream.type,
         stream.mimeType, streamInfo.mediaSequenceToStartTime, mediaVariables,
-        stream.codecs, stream.bandwidth);
+        stream.codecs, stream.bandwidth, streamInfo.streamStartTime).segments;
 
     stream.segmentIndex.mergeAndEvict(
         segments, this.presentationTimeline_.getSegmentAvailabilityStart());
@@ -351,6 +362,36 @@ shaka.hls.HlsParser = class {
     // No-op
   }
 
+  /**
+   * If necessary, makes sure that sync times will be normalized to 0, so that
+   * a stream does not start buffering at 50 years in because sync times are
+   * measured in time since 1970.
+   * @private
+   */
+  calculateSyncTimeOffset_() {
+    if (this.syncTimeOffset_ != null) {
+      // The offset was already calculated.
+      return;
+    }
+
+    const segments = new Set();
+    let lowestSyncTime = Infinity;
+    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
+      for (const segment of (streamInfo.stream.segmentIndex || [])) {
+        if (segment.syncTime != null) {
+          lowestSyncTime = Math.min(lowestSyncTime, segment.syncTime);
+          segments.add(segment);
+        }
+      }
+    }
+    if (segments.size > 0) {
+      this.syncTimeOffset_ = -lowestSyncTime;
+      for (const segment of segments) {
+        segment.syncTime += this.syncTimeOffset_;
+      }
+    }
+  }
+
   /**
    * Parses the manifest.
    *
@@ -433,6 +474,10 @@ shaka.hls.HlsParser = class {
           shaka.util.Error.Code.OPERATION_ABORTED);
     }
 
+    // Now that we have generated all streams, we can determine the offset to
+    // apply to sync times.
+    this.calculateSyncTimeOffset_();
+
     if (this.aesEncrypted_ && variants.length == 0) {
       // We do not support AES-128 encryption with HLS yet. Variants is null
       // when the playlist is encrypted with AES-128.
@@ -1451,10 +1496,13 @@ shaka.hls.HlsParser = class {
     const mediaSequenceToStartTime = new Map();
 
     let segments;
+    let streamStartTime = 0;
     try {
-      segments = this.createSegments_(verbatimMediaPlaylistUri,
+      const ret = this.createSegments_(verbatimMediaPlaylistUri,
           playlist, type, mimeType, mediaSequenceToStartTime, mediaVariables,
           codecs, bandwidth);
+      segments = ret.segments;
+      streamStartTime = ret.streamStartTime;
     } catch (error) {
       if (error.code == shaka.util.Error.Code.HLS_INTERNAL_SKIP_STREAM) {
         shaka.log.alwaysWarn('Skipping unsupported HLS stream',
@@ -1527,6 +1575,7 @@ shaka.hls.HlsParser = class {
       maxTimestamp: lastEndTime,
       mediaSequenceToStartTime,
       canSkipSegments,
+      streamStartTime,
     };
   }
 
@@ -1706,8 +1755,8 @@ shaka.hls.HlsParser = class {
    * @param {number} startTime
    * @param {!Map.<string, string>} variables
    * @param {string} absoluteMediaPlaylistUri
-   * @return {!shaka.media.SegmentReference}
    * @param {string} type
+   * @return {!shaka.media.SegmentReference}
    * @private
    */
   createSegmentReference_(
@@ -1730,6 +1779,24 @@ shaka.hls.HlsParser = class {
                 'true, and see https://bit.ly/3clctcj for details.');
     }
 
+    let syncTime = null;
+    if (!this.config_.hls.ignoreManifestProgramDateTime) {
+      const dateTimeTag =
+          shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-PROGRAM-DATE-TIME');
+      if (dateTimeTag && dateTimeTag.value) {
+        const time = shaka.util.XmlUtils.parseDate(
+            dateTimeTag.value, /* subSecond= */ true);
+        goog.asserts.assert(time != null,
+            'EXT-X-PROGRAM-DATE-TIME format not valid');
+        // Sync time offset is null on the first go-through. This indicates that
+        // we have not yet seen every stream, and thus do not yet have enough
+        // information to determine how to normalize the sync times.
+        // For that first go-through, the sync time will be applied after the
+        // references are all created. Until then, just offset by 0.
+        syncTime = time + (this.syncTimeOffset_ || 0);
+      }
+    }
+
     // Create SegmentReferences for the partial segments.
     const partialSegmentRefs = [];
     if (this.lowLatencyMode_ && hlsSegment.partialSegments.length) {
@@ -1839,6 +1906,7 @@ shaka.hls.HlsParser = class {
         partialSegmentRefs,
         tilesLayout,
         tileDuration,
+        syncTime,
     );
   }
 
@@ -1894,11 +1962,15 @@ shaka.hls.HlsParser = class {
    * @param {!Map.<string, string>} variables
    * @param {string} codecs
    * @param {(number|undefined)} bandwidth
-   * @return {!Array.<!shaka.media.SegmentReference>}
+   * @param {number=} streamStartTime
+   * @return {{
+   *   segments: !Array.<!shaka.media.SegmentReference>,
+   *   streamStartTime: number
+   * }}
    * @private
    */
   createSegments_(verbatimMediaPlaylistUri, playlist, type, mimeType,
-      mediaSequenceToStartTime, variables, codecs, bandwidth) {
+      mediaSequenceToStartTime, variables, codecs, bandwidth, streamStartTime) {
     /** @type {Array.<!shaka.hls.Segment>} */
     const hlsSegments = playlist.segments;
     goog.asserts.assert(hlsSegments.length, 'Playlist should have segments!');
@@ -1910,13 +1982,22 @@ shaka.hls.HlsParser = class {
     // time.
     const mediaSequenceNumber = shaka.hls.Utils.getFirstTagWithNameAsNumber(
         playlist.tags, 'EXT-X-MEDIA-SEQUENCE', 0);
+    const targetDuration = shaka.hls.Utils.getFirstTagWithNameAsNumber(
+        playlist.tags, 'EXT-X-TARGETDURATION', 0);
     const skipTag = shaka.hls.Utils.getFirstTagWithName(playlist.tags,
         'EXT-X-SKIP');
     const skippedSegments =
         skipTag ? Number(skipTag.getAttributeValue('SKIPPED-SEGMENTS')) : 0;
     let position = mediaSequenceNumber + skippedSegments;
     let firstStartTime = 0;
 
+    if (!streamStartTime) {
+      // Make an approximation of the start time of the first segment. This is
+      // used to synthesize program date-time values if there are none, to
+      // handle the case of different streams starting at different times.
+      streamStartTime = mediaSequenceNumber * targetDuration;
+    }
+
     // For live stream, use the cached value in the mediaSequenceToStartTime
     // map if available.
     if (this.isLive_() && mediaSequenceToStartTime.has(position)) {
@@ -1928,11 +2009,11 @@ shaka.hls.HlsParser = class {
         'starts at', firstStartTime);
 
     /** @type {!Array.<!shaka.media.SegmentReference>} */
-    const references = [];
+    const segments = [];
 
     const enumerate = (it) => shaka.util.Iterables.enumerate(it);
     for (const {i, item} of enumerate(hlsSegments)) {
-      const previousReference = references[references.length - 1];
+      const previousReference = segments[segments.length - 1];
       const startTime =
           (i == 0) ? firstStartTime : previousReference.endTime;
       position = mediaSequenceNumber + skippedSegments + i;
@@ -1963,10 +2044,58 @@ shaka.hls.HlsParser = class {
           playlist.absoluteUri,
           type);
 
-      references.push(reference);
+      segments.push(reference);
+    }
+
+    // If some segments have sync times, but not all, extrapolate the sync
+    // times of the ones with none.
+    const someSyncTime = segments.some((ref) => ref.syncTime != null);
+    if (someSyncTime) {
+      for (let i = 0; i < segments.length; i++) {
+        const reference = segments[i];
+        if (reference.syncTime != null) {
+          // No need to extrapolate.
+          continue;
+        }
+        // Find the nearest segment with syncTime, in either direction.
+        // This looks forward and backward simultaneously, keeping track of what
+        // to offset the syncTime it finds by as it goes.
+        let forwardAdd = 0;
+        let backwardAdd = 0;
+        for (let j = 0; reference.syncTime == null; j++) {
+          for (const forward of [false, true]) {
+            const other = segments[i + j * (forward ? 1 : -1)];
+            if (other) {
+              if (other.syncTime != null) {
+                reference.syncTime = other.syncTime;
+                reference.syncTime += forward ? forwardAdd : backwardAdd;
+                break;
+              }
+              if (forward) {
+                forwardAdd -= other.endTime - other.startTime;
+              }
+              if (!forward) {
+                backwardAdd += other.endTime - other.startTime;
+              }
+            }
+          }
+        }
+      }
+    } else {
+      // If we want to use program date-time for sync-ing, but there is no such
+      // tag anywhere in the manifest, create synthetic sync times based on the
+      // start time of the manifest.
+      for (const reference of segments) {
+        reference.syncTime = reference.startTime;
+        // Modify this sync time by the start time of the playlist, in case
+        // different streams start at different times.
+        reference.syncTime += streamStartTime;
+        // Also apply the offset, if present.
+        reference.syncTime += this.syncTimeOffset_ || 0;
+      }
     }
 
-    return references;
+    return {segments, streamStartTime};
   }
 
   /**
@@ -2343,7 +2472,8 @@ shaka.hls.HlsParser = class {
  *   minTimestamp: number,
  *   maxTimestamp: number,
  *   mediaSequenceToStartTime: !Map.<number, number>,
- *   canSkipSegments: boolean
+ *   canSkipSegments: boolean,
+ *   streamStartTime: number
  * }}
  *
  * @description
@@ -2368,6 +2498,9 @@ shaka.hls.HlsParser = class {
  * @property {boolean} canSkipSegments
  *  True if the server supports delta playlist updates, and we can send a
  *  request for a playlist that can skip older media segments.
+ * @property {number} streamStartTime
+ *  The time of the first segment of the stream. Used to synthesize program
+ *  date-time values if none are present, for audio-video synchronization.
  */
 shaka.hls.HlsParser.StreamInfo;
 

lib/media/media_source_engine.js

@@ -506,7 +506,6 @@ shaka.media.MediaSourceEngine = class {
    * @param {?number} endTime relative to the start of the presentation
    * @param {?boolean} hasClosedCaptions True if the buffer contains CEA closed
    * captions
-
    * @param {boolean=} seeked True if we just seeked
    * @param {boolean=} sequenceMode True if sequence mode
    * @return {!Promise}
@@ -515,11 +514,11 @@ shaka.media.MediaSourceEngine = class {
       seeked, sequenceMode) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
 
-    // If we just cleared buffer and is on an unbuffered seek, we need to set
-    // the new timestampOffset of the sourceBuffer.
-    // Don't do this for text streams, though, since they don't use MediaSource
-    // anyway.
     if (startTime != null && sequenceMode && contentType != ContentType.TEXT) {
+      // If we just cleared buffer and is on an unbuffered seek, we need to set
+      // the new timestampOffset of the sourceBuffer.
+      // Don't do this for text streams, though, since they don't use
+      // MediaSource anyway.
       if (seeked) {
         const timestampOffset = /** @type {number} */ (startTime);
         this.enqueueOperation_(

lib/media/segment_index.js

@@ -348,7 +348,8 @@ shaka.media.SegmentIndex = class {
             lastReference.appendWindowEnd,
             lastReference.partialReferences,
             lastReference.tilesLayout,
-            lastReference.tileDuration);
+            lastReference.tileDuration,
+            lastReference.syncTime);
   }