From 631b1fbf24bcb59c9f5cc57b068e4d2ad3ab9bc0 Mon Sep 17 00:00:00 2001
From: Theodore Abshire <theodab@google.com>
Date: Sun, 11 Sep 2022 15:35:02 -0700
Subject: [PATCH] feat(HLS): Lazy-load HLS media playlists

This changes the HLS parser so that the media playlists are only
downloaded when the createSegmentIndex function for the associated
stream is called.
Because there is some important information about HLS streams that
is stored inside the media playlist, this also changes the player
to call createSegmentIndex on the initial variant earlier in the
load process, to make sure that information is available in time.

Closes #1936
---
 lib/hls/hls_parser.js              | 781 ++++++++++++++++++-----------
 lib/media/presentation_timeline.js |  14 +
 lib/player.js                      |  63 ++-
 test/hls/hls_live_unit.js          | 106 +++-
 test/hls/hls_parser_unit.js        |  68 ++-
 5 files changed, 699 insertions(+), 333 deletions(-)

diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js
index 26801b272b..ce5442a52f 100644
--- a/lib/hls/hls_parser.js
+++ b/lib/hls/hls_parser.js
@@ -71,6 +71,22 @@ shaka.hls.HlsParser = class {
      */
     this.groupIdToStreamInfosMap_ = new Map();
 
+    /**
+     * For media playlist lazy-loading to work in livestreams, we have to assume
+     * that each stream of a type (video, audio, etc) has the same mappings of
+     * sequence number to start time.
+     * This map stores those relationships.
+     * @private {!Map.<string, !Map.<number, number>>}
+     */
+    this.mediaSequenceToStartTimeByType_ = new Map();
+
+    // Set initial maps.
+    const ContentType = shaka.util.ManifestParserUtils.ContentType;
+    this.mediaSequenceToStartTimeByType_.set(ContentType.VIDEO, new Map());
+    this.mediaSequenceToStartTimeByType_.set(ContentType.AUDIO, new Map());
+    this.mediaSequenceToStartTimeByType_.set(ContentType.TEXT, new Map());
+    this.mediaSequenceToStartTimeByType_.set(ContentType.IMAGE, new Map());
+
     /**
      * The values are strings of the form "<VIDEO URI> - <AUDIO URI>",
      * where the URIs are the verbatim media playlist URIs as they appeared in
@@ -122,19 +138,20 @@ shaka.hls.HlsParser = class {
     this.updatePlaylistDelay_ = 0;
 
     /**
-     * If true, we have already calculated offsets to synchronize streams.
-     * Offsets are computed in syncStreams*_().
-     * @private {boolean}
+     * The minimum sequence number for generated segments, when ignoring
+     * EXT-X-PROGRAM-DATE-TIME.
+     *
+     * @private {number}
      */
-    this.streamsSynced_ = false;
+    this.minSequenceNumber_ = -1;
 
     /**
-     * The minimum sequence number for generated segments, when ignoring
-     * EXT-X-PROGRAM-DATE-TIME.
+     * The lowest sync number for streams. Used for synchronizing via
+     * EXT-X-PROGRAM-DATE-TIME
      *
-     * @private {?number}
+     * @private {number}
      */
-    this.minSequenceNumber_ = null;
+    this.lowestSyncTime_ = Infinity;
 
     /**
      * This timer is used to trigger the start of a manifest update. A manifest
@@ -171,9 +188,6 @@ shaka.hls.HlsParser = class {
     /** @private {shaka.util.OperationManager} */
     this.operationManager_ = new shaka.util.OperationManager();
 
-    /** @private {!Array.<!Array.<!shaka.media.SegmentReference>>} */
-    this.segmentsToNotifyByStream_ = [];
-
     /** A map from closed captions' group id, to a map of closed captions info.
      * {group id -> {closed captions channel id -> language}}
      * @private {Map.<string, Map.<string, string>>}
@@ -274,22 +288,20 @@ shaka.hls.HlsParser = class {
     /** @type {!Array.<!Promise>} */
     const updates = [];
     const streamInfos = Array.from(this.uriToStreamInfosMap_.values());
-    // Wait for the first stream info created, so that the start time is fetched
-    // and can be reused.
-    if (streamInfos.length) {
-      await this.updateStream_(streamInfos[0]);
-    }
-    for (let i = 1; i < streamInfos.length; i++) {
-      updates.push(this.updateStream_(streamInfos[i]));
-    }
 
+    // Only update active streams.
+    const activeStreamInfos = streamInfos.filter((s) => s.stream.segmentIndex);
+    for (const streamInfo of activeStreamInfos) {
+      updates.push(this.updateStream_(streamInfo));
+    }
     await Promise.all(updates);
 
-    this.notifySegments_();
+    // Now that streams have been updated, notify the presentation timeline.
+    this.notifySegmentsForStreams_(activeStreamInfos.map((s) => s.stream));
 
     // If any hasEndList is false, the stream is still live.
-    const stillLive = streamInfos.some((s) => s.hasEndList == false);
-    if (!stillLive) {
+    const stillLive = activeStreamInfos.some((s) => s.hasEndList == false);
+    if (activeStreamInfos.length && !stillLive) {
       // Convert the presentation to VOD and set the duration.
       const PresentationType = shaka.hls.HlsParser.PresentationType_;
       this.setPresentationType_(PresentationType.VOD);
@@ -337,18 +349,18 @@ shaka.hls.HlsParser = class {
 
     const stream = streamInfo.stream;
 
+    const mediaSequenceToStartTime =
+        this.mediaSequenceToStartTimeByType_.get(streamInfo.type);
     const segments = this.createSegments_(
         streamInfo.verbatimMediaPlaylistUri, playlist, stream.type,
-        stream.mimeType, streamInfo.mediaSequenceToStartTime, mediaVariables,
-        stream.codecs);
-    this.segmentsToNotifyByStream_.push(segments);
+        stream.mimeType, mediaSequenceToStartTime, mediaVariables);
 
     stream.segmentIndex.mergeAndEvict(
         segments, this.presentationTimeline_.getSegmentAvailabilityStart());
     if (segments.length) {
       const mediaSequenceNumber = shaka.hls.Utils.getFirstTagWithNameAsNumber(
           playlist.tags, 'EXT-X-MEDIA-SEQUENCE', 0);
-      const playlistStartTime = streamInfo.mediaSequenceToStartTime.get(
+      const playlistStartTime = mediaSequenceToStartTime.get(
           mediaSequenceNumber);
       stream.segmentIndex.evict(playlistStartTime);
     }
@@ -379,68 +391,67 @@ shaka.hls.HlsParser = class {
   }
 
   /**
-   * Align all streams by sequence number by dropping early segments.  Then
-   * offset all streams to begin at presentation time 0.
+   * Align the streams by sequence number by dropping early segments.  Then
+   * offset the streams to begin at presentation time 0.
+   * @param {!Array.<!shaka.hls.HlsParser.StreamInfo>} streamInfos
    * @private
    */
-  syncStreamsWithSequenceNumber_() {
-    if (this.streamsSynced_) {
-      return;
-    }
-
+  syncStreamsWithSequenceNumber_(streamInfos) {
+    // We assume that, when this is first called, we have enough info to
+    // determine how to use the program date times (e.g. we have both a video
+    // and an audio, and all other videos and audios match those).
+    // Thus, we only need to calculate this once.
+    const updateMinSequenceNumber = this.minSequenceNumber_ == -1;
     // Sync using media sequence number.  Find the highest starting sequence
     // number among all streams.  Later, we will drop any references to
     // earlier segments in other streams, then offset everything back to 0.
-    let highestStartingSequenceNumber = -1;
-    const firstSequenceNumberMap = new Map();
-
-    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
+    for (const streamInfo of streamInfos) {
       const segmentIndex = streamInfo.stream.segmentIndex;
-      if (segmentIndex) {
-        const segment0 = segmentIndex.earliestReference();
-        if (segment0) {
-          // This looks inefficient, but iteration order is insertion order.
-          // So the very first entry should be the one we want.
-          // We assert that this holds true so that we are alerted by debug
-          // builds and tests if it changes.  We still do a loop, though, so
-          // that the code functions correctly in production no matter what.
-          if (goog.DEBUG) {
-            const firstSequenceStartTime =
-                streamInfo.mediaSequenceToStartTime.values().next().value;
-            goog.asserts.assert(
-                firstSequenceStartTime == segment0.startTime,
-                'Sequence number map is not ordered as expected!');
-          }
-          for (const [sequence, start] of streamInfo.mediaSequenceToStartTime) {
-            if (start == segment0.startTime) {
-              firstSequenceNumberMap.set(streamInfo, sequence);
-
-              highestStartingSequenceNumber = Math.max(
-                  highestStartingSequenceNumber, sequence);
-              break;
+      goog.asserts.assert(segmentIndex,
+          'Only loaded streams should be synced');
+      const mediaSequenceToStartTime =
+          this.mediaSequenceToStartTimeByType_.get(streamInfo.type);
+      const segment0 = segmentIndex.earliestReference();
+      if (segment0) {
+        // This looks inefficient, but iteration order is insertion order.
+        // So the very first entry should be the one we want.
+        // We assert that this holds true so that we are alerted by debug
+        // builds and tests if it changes.  We still do a loop, though, so
+        // that the code functions correctly in production no matter what.
+        if (goog.DEBUG) {
+          const firstSequenceStartTime =
+              mediaSequenceToStartTime.values().next().value;
+          goog.asserts.assert(
+              firstSequenceStartTime == segment0.startTime,
+              'Sequence number map is not ordered as expected!');
+        }
+        for (const [sequence, start] of mediaSequenceToStartTime) {
+          if (start == segment0.startTime) {
+            streamInfo.firstSequenceNumber = sequence;
+            if (updateMinSequenceNumber) {
+              this.minSequenceNumber_ = Math.max(
+                  this.minSequenceNumber_, sequence);
             }
+            break;
           }
         }
       }
     }
 
-    if (highestStartingSequenceNumber < 0) {
+    if (this.minSequenceNumber_ < 0) {
       // Nothing to sync.
       return;
     }
 
-    // From now on, updates will ignore any references before this number.
-    this.minSequenceNumber_ = highestStartingSequenceNumber;
-
     shaka.log.debug('Syncing HLS streams against base sequence number:',
         this.minSequenceNumber_);
 
-    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
+    for (const streamInfo of streamInfos) {
       const segmentIndex = streamInfo.stream.segmentIndex;
       if (segmentIndex) {
         // Drop any earlier references.
-        const numSegmentsToDrop = this.minSequenceNumber_ -
-            firstSequenceNumberMap.get(streamInfo);
+        const numSegmentsToDrop =
+            this.minSequenceNumber_ - streamInfo.firstSequenceNumber;
         segmentIndex.dropFirstReferences(numSegmentsToDrop);
 
         // Now adjust timestamps back to begin at 0.
@@ -450,33 +461,34 @@ shaka.hls.HlsParser = class {
         }
       }
     }
-
-    this.streamsSynced_ = true;
   }
 
   /**
    * Synchronize streams by the EXT-X-PROGRAM-DATE-TIME tags attached to their
    * segments.  Also normalizes segment times so that the earliest segment in
-   * any stream is at time 0.
+   * the stream is at time 0.
+   * @param {!Array.<!shaka.hls.HlsParser.StreamInfo>} streamInfos
    * @private
    */
-  syncStreamsWithProgramDateTime_() {
-    if (this.streamsSynced_) {
-      return;
-    }
-
-    let lowestSyncTime = Infinity;
-
-    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
-      const segmentIndex = streamInfo.stream.segmentIndex;
-      if (segmentIndex) {
+  syncStreamsWithProgramDateTime_(streamInfos) {
+    // We assume that, when this is first called, we have enough info to
+    // determine how to use the program date times (e.g. we have both a video
+    // and an audio, and all other videos and audios match those).
+    // Thus, we only need to calculate this once.
+    if (this.lowestSyncTime_ == Infinity) {
+      for (const streamInfo of streamInfos) {
+        const segmentIndex = streamInfo.stream.segmentIndex;
+        goog.asserts.assert(segmentIndex,
+            'Only loaded streams should be synced');
         const segment0 = segmentIndex.earliestReference();
         if (segment0 != null && segment0.syncTime != null) {
-          lowestSyncTime = Math.min(lowestSyncTime, segment0.syncTime);
+          this.lowestSyncTime_ =
+              Math.min(this.lowestSyncTime_, segment0.syncTime);
         }
       }
     }
 
+    const lowestSyncTime = this.lowestSyncTime_;
     if (lowestSyncTime == Infinity) {
       // Nothing to sync.
       return;
@@ -506,8 +518,6 @@ shaka.hls.HlsParser = class {
         }
       }
     }
-
-    this.streamsSynced_ = true;
   }
 
   /**
@@ -522,8 +532,10 @@ shaka.hls.HlsParser = class {
     goog.asserts.assert(streamInfo.maxTimestamp >= 0,
         'Negative maxTimestamp after adjustment!');
 
-    for (const [key, value] of streamInfo.mediaSequenceToStartTime) {
-      streamInfo.mediaSequenceToStartTime.set(key, value + offset);
+    const mediaSequenceToStartTime =
+        this.mediaSequenceToStartTimeByType_.get(streamInfo.type);
+    for (const [key, value] of mediaSequenceToStartTime) {
+      mediaSequenceToStartTime.set(key, value + offset);
     }
 
     shaka.log.debug('Offset', offset, 'applied to',
@@ -641,10 +653,10 @@ shaka.hls.HlsParser = class {
 
       // Parse audio and video media tags first, so that we can extract segment
       // start time from audio/video streams and reuse for text streams.
-      await this.createStreamInfosFromMediaTags_(mediaTags);
+      this.createStreamInfosFromMediaTags_(mediaTags);
       this.parseClosedCaptions_(mediaTags);
-      variants = await this.createVariantsForTags_(variantTags);
-      textStreams = await this.parseTexts_(mediaTags);
+      variants = this.createVariantsForTags_(variantTags);
+      textStreams = this.parseTexts_(mediaTags);
       imageStreams = await this.parseImages_(imageTags);
     }
 
@@ -656,32 +668,32 @@ shaka.hls.HlsParser = class {
           shaka.util.Error.Code.OPERATION_ABORTED);
     }
 
-    // Now that we have generated all streams, we can determine the offset to
-    // apply to sync times.
-    if (this.config_.hls.ignoreManifestProgramDateTime) {
-      this.syncStreamsWithSequenceNumber_();
-    } else {
-      this.syncStreamsWithProgramDateTime_();
-    }
-
-    // Find the min and max timestamp of the earliest segment in all streams.
-    // Find the minimum duration of all streams as well.
-    let minDuration = Infinity;
-    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
-      if (streamInfo.stream.type != 'text') {
-        // Since everything is already offset to 0 (either by sync or by being
-        // VOD), only maxTimestamp is necessary to compute the duration.
-        minDuration = Math.min(minDuration, streamInfo.maxTimestamp);
-      }
-    }
-
     // This assert is our own sanity check.
     goog.asserts.assert(this.presentationTimeline_ == null,
         'Presentation timeline created early!');
-    this.createPresentationTimeline_();
+    // We don't know if the presentation is VOD or live until we parse at least
+    // one media playlist, so make a VOD-style presentation timeline for now
+    // and change the type later if we discover this is live.
+    // Since the player will load the first variant chosen early in the process,
+    // there isn't a window during playback where the live-ness is unknown.
+    this.presentationTimeline_ = new shaka.media.PresentationTimeline(
+    /* presentationStartTime= */ null, /* delay= */ 0);
+    this.presentationTimeline_.setStatic(true);
+
+    this.manifest_ = {
+      presentationTimeline: this.presentationTimeline_,
+      variants,
+      textStreams,
+      imageStreams,
+      offlineSessionIds: [],
+      minBufferTime: 0,
+      sequenceMode: true,
+    };
+    this.playerInterface_.makeTextStreamsForClosedCaptions(this.manifest_);
+  }
 
-    // This assert satisfies the compiler that it is not null for the rest of
-    // the method.
+  /** @private */
+  determineDuration_() {
     goog.asserts.assert(this.presentationTimeline_,
         'Presentation timeline not created!');
 
@@ -716,33 +728,9 @@ shaka.hls.HlsParser = class {
       }
     } else {
       // Use the minimum duration as the presentation duration.
-      this.presentationTimeline_.setDuration(minDuration);
-
-      for (const streamInfo of this.uriToStreamInfosMap_.values()) {
-        // Fit the segments to the playlist duration.
-        streamInfo.stream.segmentIndex.fit(/* periodStart= */ 0, minDuration);
-      }
-    }
-
-    // Now that the content has been fit, notify segments.
-    this.segmentsToNotifyByStream_ = [];
-    const streamsToNotify = [];
-    for (const variant of variants) {
-      for (const stream of [variant.video, variant.audio]) {
-        if (stream) {
-          streamsToNotify.push(stream);
-        }
-      }
-    }
-    await Promise.all(streamsToNotify.map(async (stream) => {
-      await stream.createSegmentIndex();
-    }));
-    for (const stream of streamsToNotify) {
-      this.segmentsToNotifyByStream_.push(stream.segmentIndex.references);
+      this.presentationTimeline_.setDuration(this.getMinDuration_());
     }
 
-    this.notifySegments_();
-
     // This is the first point where we have a meaningful presentation start
     // time, and we need to tell PresentationTimeline that so that it can
     // maintain consistency from here on.
@@ -753,17 +741,6 @@ shaka.hls.HlsParser = class {
     goog.asserts.assert(
         !this.presentationTimeline_.usingPresentationStartTime(),
         'We should not be using the presentation start time in HLS!');
-
-    this.manifest_ = {
-      presentationTimeline: this.presentationTimeline_,
-      variants,
-      textStreams,
-      imageStreams,
-      offlineSessionIds: [],
-      minBufferTime: 0,
-      sequenceMode: true,
-    };
-    this.playerInterface_.makeTextStreamsForClosedCaptions(this.manifest_);
   }
 
   /**
@@ -848,21 +825,20 @@ shaka.hls.HlsParser = class {
    * Create text streams for Subtitles, but not Closed Captions.
    *
    * @param {!Array.<!shaka.hls.Tag>} mediaTags Media tags from the playlist.
-   * @return {!Promise.<!Array.<!shaka.extern.Stream>>}
+   * @return {!Array.<!shaka.extern.Stream>}
    * @private
    */
-  async parseTexts_(mediaTags) {
+  parseTexts_(mediaTags) {
     // Create text stream for each Subtitle media tag.
     const subtitleTags =
         shaka.hls.Utils.filterTagsByType(mediaTags, 'SUBTITLES');
-    const textStreamPromises = subtitleTags.map(async (tag) => {
+    const textStreams = subtitleTags.map((tag) => {
       const disableText = this.config_.disableText;
       if (disableText) {
         return null;
       }
       try {
-        const streamInfo = await this.createStreamInfoFromMediaTag_(tag);
-        return streamInfo.stream;
+        return this.createStreamInfoFromMediaTag_(tag).stream;
       } catch (e) {
         if (this.config_.hls.ignoreTextStreamFailures) {
           return null;
@@ -870,7 +846,6 @@ shaka.hls.HlsParser = class {
         throw e;
       }
     });
-    const textStreams = await Promise.all(textStreamPromises);
 
     // Set the codecs for text streams.
     for (const tag of subtitleTags) {
@@ -920,7 +895,7 @@ shaka.hls.HlsParser = class {
    * @param {!Array.<!shaka.hls.Tag>} mediaTags Media tags from the playlist.
    * @private
    */
-  async createStreamInfosFromMediaTags_(mediaTags) {
+  createStreamInfosFromMediaTags_(mediaTags) {
     // Filter out subtitles and  media tags without uri.
     mediaTags = mediaTags.filter((tag) => {
       const uri = tag.getAttributeValue('URI') || '';
@@ -929,20 +904,19 @@ shaka.hls.HlsParser = class {
     });
 
     // Create stream info for each audio / video media tag.
-    const promises = mediaTags.map((tag) => {
-      return this.createStreamInfoFromMediaTag_(tag);
-    });
-    await Promise.all(promises);
+    for (const tag of mediaTags) {
+      this.createStreamInfoFromMediaTag_(tag);
+    }
   }
 
   /**
    * @param {!Array.<!shaka.hls.Tag>} tags Variant tags from the playlist.
-   * @return {!Promise.<!Array.<!shaka.extern.Variant>>}
+   * @return {!Array.<!shaka.extern.Variant>}
    * @private
    */
-  async createVariantsForTags_(tags) {
+  createVariantsForTags_(tags) {
     // Create variants for each variant tag.
-    const variantsPromises = tags.map(async (tag) => {
+    const allVariants = tags.map((tag) => {
       const frameRate = tag.getAttributeValue('FRAME-RATE');
       const bandwidth = Number(tag.getAttributeValue('AVERAGE-BANDWIDTH')) ||
         Number(tag.getRequiredAttrValue('BANDWIDTH'));
@@ -952,7 +926,7 @@ shaka.hls.HlsParser = class {
 
       const videoRange = tag.getAttributeValue('VIDEO-RANGE');
 
-      const streamInfos = await this.createStreamInfosForVariantTag_(tag,
+      const streamInfos = this.createStreamInfosForVariantTag_(tag,
           resolution, frameRate);
 
       goog.asserts.assert(streamInfos.audio.length ||
@@ -967,8 +941,6 @@ shaka.hls.HlsParser = class {
           frameRate,
           videoRange);
     });
-
-    const allVariants = await Promise.all(variantsPromises);
     let variants = allVariants.reduce(shaka.util.Functional.collapseArrays, []);
     // Filter out null variants.
     variants = variants.filter((variant) => variant != null);
@@ -982,10 +954,10 @@ shaka.hls.HlsParser = class {
    * @param {!shaka.hls.Tag} tag
    * @param {?string} resolution
    * @param {?string} frameRate
-   * @return {!Promise.<!shaka.hls.HlsParser.StreamInfos>}
+   * @return {!shaka.hls.HlsParser.StreamInfos}
    * @private
    */
-  async createStreamInfosForVariantTag_(tag, resolution, frameRate) {
+  createStreamInfosForVariantTag_(tag, resolution, frameRate) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
     /** @type {!Array.<string>} */
     let allCodecs = this.getCodecsForVariantTag_(tag);
@@ -1054,7 +1026,7 @@ shaka.hls.HlsParser = class {
 
     if (!ignoreStream) {
       const streamInfo =
-          await this.createStreamInfoFromVariantTag_(tag, allCodecs, type);
+          this.createStreamInfoFromVariantTag_(tag, allCodecs, type);
       res[streamInfo.stream.type] = [streamInfo];
     }
     this.filterLegacyCodecs_(res);
@@ -1188,7 +1160,7 @@ shaka.hls.HlsParser = class {
 
   /**
    * Get the type value.
-   * Shaka recognizes the content types 'audio', 'video' and 'text'.
+   * Shaka recognizes the content types 'audio', 'video', 'text', and 'image'.
    * The HLS 'subtitles' type needs to be mapped to 'text'.
    * @param {!shaka.hls.Tag} tag
    * @return {string}
@@ -1347,10 +1319,10 @@ shaka.hls.HlsParser = class {
    * Parse EXT-X-MEDIA media tag into a Stream object.
    *
    * @param {shaka.hls.Tag} tag
-   * @return {!Promise.<!shaka.hls.HlsParser.StreamInfo>}
+   * @return {!shaka.hls.HlsParser.StreamInfo}
    * @private
    */
-  async createStreamInfoFromMediaTag_(tag) {
+  createStreamInfoFromMediaTag_(tag) {
     goog.asserts.assert(tag.name == 'EXT-X-MEDIA',
         'Should only be called on media tags!');
     const groupId = tag.getRequiredAttrValue('GROUP-ID');
@@ -1392,7 +1364,7 @@ shaka.hls.HlsParser = class {
     const forced = forcedAttrValue == 'YES';
     // TODO: Should we take into account some of the currently ignored
     // attributes: INSTREAM-ID, Attribute descriptions: https://bit.ly/2lpjOhj
-    const streamInfo = await this.createStreamInfo_(
+    const streamInfo = this.createStreamInfo_(
         verbatimMediaPlaylistUri, codecs, type, language, primary, name,
         channelsCount, /* closedCaptions= */ null, characteristics, forced,
         spatialAudio);
@@ -1439,7 +1411,7 @@ shaka.hls.HlsParser = class {
 
     const characteristics = tag.getAttributeValue('CHARACTERISTICS');
 
-    const streamInfo = await this.createStreamInfo_(
+    const streamInfo = this.createStreamInfo_(
         verbatimImagePlaylistUri, codecs, type, language, /* primary= */ false,
         name, /* channelsCount= */ null, /* closedCaptions= */ null,
         characteristics, /* forced= */ false, /* spatialAudio= */ false);
@@ -1457,6 +1429,10 @@ shaka.hls.HlsParser = class {
       // of the entire sheet at once (like we expect in the output).
       // So multiply by the layout size.
 
+      // Since we need to have generated the segment index for this, we can't
+      // lazy-load in this situation.
+      await streamInfo.stream.createSegmentIndex();
+
       const reference = streamInfo.stream.segmentIndex.get(0);
       const layout = reference.getTilesLayout();
       if (layout) {
@@ -1483,10 +1459,10 @@ shaka.hls.HlsParser = class {
    * @param {!shaka.hls.Tag} tag
    * @param {!Array.<string>} allCodecs
    * @param {string} type
-   * @return {!Promise.<!shaka.hls.HlsParser.StreamInfo>}
+   * @return {!shaka.hls.HlsParser.StreamInfo}
    * @private
    */
-  async createStreamInfoFromVariantTag_(tag, allCodecs, type) {
+  createStreamInfoFromVariantTag_(tag, allCodecs, type) {
     goog.asserts.assert(tag.name == 'EXT-X-STREAM-INF',
         'Should only be called on variant tags!');
     const verbatimMediaPlaylistUri = this.variableSubstitution_(
@@ -1498,7 +1474,7 @@ shaka.hls.HlsParser = class {
 
     const closedCaptions = this.getClosedCaptions_(tag, type);
     const codecs = shaka.util.ManifestParserUtils.guessCodecs(type, allCodecs);
-    const streamInfo = await this.createStreamInfo_(verbatimMediaPlaylistUri,
+    const streamInfo = this.createStreamInfo_(verbatimMediaPlaylistUri,
         codecs, type, /* language= */ 'und', /* primary= */ false,
         /* name= */ null, /* channelcount= */ null, closedCaptions,
         /* characteristics= */ null, /* forced= */ false,
@@ -1526,29 +1502,191 @@ shaka.hls.HlsParser = class {
    * @param {?string} characteristics
    * @param {boolean} forced
    * @param {boolean} spatialAudio
-   * @return {!Promise.<!shaka.hls.HlsParser.StreamInfo>}
+   * @return {!shaka.hls.HlsParser.StreamInfo}
    * @private
    */
-  async createStreamInfo_(verbatimMediaPlaylistUri, codecs, type, language,
+  createStreamInfo_(verbatimMediaPlaylistUri, codecs, type, language,
       primary, name, channelsCount, closedCaptions, characteristics, forced,
       spatialAudio) {
     // TODO: Refactor, too many parameters
-    let absoluteMediaPlaylistUri = shaka.hls.Utils.constructAbsoluteUri(
+    const initialMediaPlaylistUri = shaka.hls.Utils.constructAbsoluteUri(
         this.masterPlaylistUri_, verbatimMediaPlaylistUri);
 
-    const response = await this.requestManifest_(absoluteMediaPlaylistUri);
-    // Record the final URI after redirects.
-    absoluteMediaPlaylistUri = response.uri;
+    // This stream is lazy-loaded inside the createSegmentIndex function.
+    // So we start out with a stream object that does not contain the actual
+    // segment index, then download when createSegmentIndex is called.
+    const stream = this.makeStreamObject_(codecs, type, language, primary, name,
+        channelsCount, closedCaptions, characteristics, forced, spatialAudio);
+    const streamInfo = {
+      stream,
+      type,
+      verbatimMediaPlaylistUri,
+      // These values are filled out or updated after lazy-loading:
+      absoluteMediaPlaylistUri: initialMediaPlaylistUri,
+      maxTimestamp: 0,
+      canSkipSegments: false,
+      hasEndList: false,
+      firstSequenceNumber: -1,
+    };
+    const downloadSegmentIndex = async () => {
+      // Un-assign this function, so that if it's called a second time we don't
+      // download twice.
+      stream.createSegmentIndex = () => Promise.resolve();
+
+      // Download the actual manifest.
+      const response = await this.requestManifest_(initialMediaPlaylistUri);
+      // Record the final URI after redirects.
+      const absoluteMediaPlaylistUri = response.uri;
+
+      // Record the redirected, final URI of this media playlist when we parse
+      // it.
+      /** @type {!shaka.hls.Playlist} */
+      const playlist = this.manifestTextParser_.parsePlaylist(
+          response.data, absoluteMediaPlaylistUri);
+
+      const wasLive = this.isLive_();
+      const realStreamInfo = await this.convertParsedPlaylistIntoStreamInfo_(
+          playlist, verbatimMediaPlaylistUri, absoluteMediaPlaylistUri, codecs,
+          type, language, primary, name, channelsCount, closedCaptions,
+          characteristics, forced, spatialAudio);
+      const realStream = realStreamInfo.stream;
+
+      if (this.isLive_() && !wasLive) {
+        // Now that we know that the presentation is live, convert the timeline
+        // to live.
+        this.changePresentationTimelineToLive_();
+      }
+
+      const hadEnoughInfoToFinalizeStreams =
+          this.hasEnoughInfoToFinalizeStreams_();
+
+      // Copy values from the real stream info to our initial one.
+      streamInfo.absoluteMediaPlaylistUri = absoluteMediaPlaylistUri;
+      streamInfo.maxTimestamp = realStreamInfo.maxTimestamp;
+      streamInfo.canSkipSegments = realStreamInfo.canSkipSegments;
+      streamInfo.hasEndList = realStreamInfo.hasEndList;
+      stream.segmentIndex = realStream.segmentIndex;
+      stream.encrypted = realStream.encrypted;
+      stream.drmInfos = realStream.drmInfos;
+      stream.keyIds = realStream.keyIds;
+      stream.kind = realStream.kind;
+      stream.roles = realStream.roles;
+      stream.mimeType = realStream.mimeType;
+
+      // MediaSource expects no codec strings combined with raw formats.
+      if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(stream.mimeType)) {
+        stream.codecs = '';
+      }
 
-    // Record the redirected, final URI of this media playlist when we parse it.
-    /** @type {!shaka.hls.Playlist} */
-    const playlist = this.manifestTextParser_.parsePlaylist(
-        response.data, absoluteMediaPlaylistUri);
+      // Add finishing touches to the stream that can only be done once we have
+      // more full context on the media as a whole.
+      if (this.hasEnoughInfoToFinalizeStreams_()) {
+        if (!hadEnoughInfoToFinalizeStreams) {
+          const streamInfos = Array.from(this.uriToStreamInfosMap_.values());
+          const activeStreamInfos =
+              streamInfos.filter((s) => s.stream.segmentIndex);
+          this.finalizeStreams_(activeStreamInfos);
+          // With the addition of this new stream, we now have enough info to
+          // figure out how long the streams should be. So process all streams
+          // we have downloaded up until this point.
+          this.determineDuration_();
+        } else {
+          this.finalizeStreams_([streamInfo]);
+        }
+      }
+    };
+    stream.createSegmentIndex = downloadSegmentIndex;
+    stream.closeSegmentIndex = () => {
+      stream.segmentIndex.release();
+      stream.segmentIndex = null;
+
+      // Now that we have closed the segment index, calling createSegmentIndex
+      // should download it again.
+      stream.createSegmentIndex = downloadSegmentIndex;
+    };
+
+    return streamInfo;
+  }
+
+  /**
+   * @return {number}
+   * @private
+   */
+  getMinDuration_() {
+    let minDuration = Infinity;
+    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
+      if (streamInfo.stream.segmentIndex && streamInfo.stream.type != 'text') {
+        // Since everything is already offset to 0 (either by sync or by being
+        // VOD), only maxTimestamp is necessary to compute the duration.
+        minDuration = Math.min(minDuration, streamInfo.maxTimestamp);
+      }
+    }
+    return minDuration;
+  }
+
+  /**
+   * @param {!Array.<!shaka.extern.Stream>} streams
+   * @private
+   */
+  notifySegmentsForStreams_(streams) {
+    const references = [];
+    for (const stream of streams) {
+      stream.segmentIndex.forEachTopLevelReference((reference) => {
+        references.push(reference);
+      });
+    }
+    this.presentationTimeline_.notifySegments(references);
+  }
+
+  /**
+   * @param {!Array.<!shaka.hls.HlsParser.StreamInfo>} streamInfos
+   * @private
+   */
+  finalizeStreams_(streamInfos) {
+    const minDuration = this.getMinDuration_();
+    if (!this.isLive_()) {
+      for (const streamInfo of streamInfos) {
+        streamInfo.stream.segmentIndex.fit(/* periodStart= */ 0, minDuration);
+      }
+    }
+    this.notifySegmentsForStreams_(streamInfos.map((s) => s.stream));
+    if (this.config_.hls.ignoreManifestProgramDateTime) {
+      this.syncStreamsWithSequenceNumber_(streamInfos);
+    } else {
+      this.syncStreamsWithProgramDateTime_(streamInfos);
+    }
+  }
 
-    return this.convertParsedPlaylistIntoStreamInfo_(playlist,
-        verbatimMediaPlaylistUri, absoluteMediaPlaylistUri, codecs, type,
-        language, primary, name, channelsCount, closedCaptions, characteristics,
-        forced, spatialAudio);
+  /**
+   * There are some values on streams that can only be set once we know about
+   * both the video and audio content, if present.
+   * This checks if there is at least one video downloaded (if the media has
+   * video), and that there is at least one audio downloaded (if the media has
+   * audio).
+   * @return {boolean}
+   * @private
+   */
+  hasEnoughInfoToFinalizeStreams_() {
+    if (!this.manifest_) {
+      return false;
+    }
+    const videos = [];
+    const audios = [];
+    for (const variant of this.manifest_.variants) {
+      if (variant.video) {
+        videos.push(variant.video);
+      }
+      if (variant.audio) {
+        audios.push(variant.audio);
+      }
+    }
+    if (videos.length > 0 && !videos.some((stream) => stream.segmentIndex)) {
+      return false;
+    }
+    if (audios.length > 0 && !audios.some((stream) => stream.segmentIndex)) {
+      return false;
+    }
+    return true;
   }
 
   /**
@@ -1655,16 +1793,10 @@ shaka.hls.HlsParser = class {
           shaka.util.Error.Code.HLS_KEYFORMATS_NOT_SUPPORTED);
     }
 
-    // MediaSource expects no codec strings combined with raw formats.
-    if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(mimeType)) {
-      codecs = '';
-    }
-
-    /** @type {!Map.<number, number>} */
-    const mediaSequenceToStartTime = new Map();
-
+    const mediaSequenceToStartTime =
+        this.mediaSequenceToStartTimeByType_.get(type);
     const segments = this.createSegments_(verbatimMediaPlaylistUri, playlist,
-        type, mimeType, mediaSequenceToStartTime, mediaVariables, codecs);
+        type, mimeType, mediaSequenceToStartTime, mediaVariables);
 
     const lastEndTime = segments[segments.length - 1].endTime;
     /** @type {!shaka.media.SegmentIndex} */
@@ -1685,18 +1817,65 @@ shaka.hls.HlsParser = class {
     const canSkipSegments = serverControlTag ?
           serverControlTag.getAttribute('CAN-SKIP-UNTIL') != null : false;
 
-    /** @type {shaka.extern.Stream} */
-    const stream = {
+    const stream = this.makeStreamObject_(codecs, type, language, primary, name,
+        channelsCount, closedCaptions, characteristics, forced, spatialAudio);
+    stream.segmentIndex = segmentIndex;
+    stream.encrypted = encrypted;
+    stream.drmInfos = drmInfos;
+    stream.keyIds = keyIds;
+    stream.kind = kind;
+    stream.roles = roles;
+    stream.mimeType = mimeType;
+
+    return {
+      stream,
+      type: '', // Not set here
+      verbatimMediaPlaylistUri,
+      absoluteMediaPlaylistUri,
+      maxTimestamp: lastEndTime,
+      canSkipSegments,
+      hasEndList: false,
+      firstSequenceNumber: -1,
+    };
+  }
+
+
+  /**
+   * Creates a stream object with the given parameters.
+   * The parameters that are passed into here are only the things that can be
+   * known without downloading the media playlist; other values must be set
+   * manually on the object after creation.
+   * @param {string} codecs
+   * @param {string} type
+   * @param {string} language
+   * @param {boolean} primary
+   * @param {?string} name
+   * @param {?number} channelsCount
+   * @param {Map.<string, string>} closedCaptions
+   * @param {?string} characteristics
+   * @param {boolean} forced
+   * @param {boolean} spatialAudio
+   * @return {!shaka.extern.Stream}
+   * @private
+   */
+  makeStreamObject_(codecs, type, language, primary, name, channelsCount,
+      closedCaptions, characteristics, forced, spatialAudio) {
+    // Fill out a "best-guess" mimeType, for now. It will be replaced once the
+    // stream is lazy-loaded.
+    const mimeType = this.guessMimeTypeBeforeLoading_(type, codecs) ||
+        this.guessMimeTypeFallback_(type);
+
+    return {
       id: this.globalId_++,
       originalId: name,
       createSegmentIndex: () => Promise.resolve(),
-      segmentIndex,
+      segmentIndex: null,
       mimeType,
       codecs,
-      kind,
-      encrypted,
-      drmInfos,
-      keyIds,
+      kind: undefined,
+      encrypted: false,
+      drmInfos: [],
+      keyIds: new Set(),
       language,
       label: name,  // For historical reasons, since before "originalId".
       type,
@@ -1709,25 +1888,15 @@ shaka.hls.HlsParser = class {
       width: undefined,
       height: undefined,
       bandwidth: undefined,
-      roles: roles,
-      forced: forced,
+      roles: [],
+      forced,
       channelsCount,
       audioSamplingRate: null,
-      spatialAudio: spatialAudio,
+      spatialAudio,
       closedCaptions,
       hdr: undefined,
       tilesLayout: undefined,
     };
-
-    return {
-      stream,
-      verbatimMediaPlaylistUri,
-      absoluteMediaPlaylistUri,
-      maxTimestamp: lastEndTime,
-      mediaSequenceToStartTime,
-      canSkipSegments,
-      hasEndList: false,
-    };
   }
 
 
@@ -1870,39 +2039,33 @@ shaka.hls.HlsParser = class {
   /**
    * @private
    */
-  createPresentationTimeline_() {
-    if (this.isLive_()) {
-      // The live edge will be calculated from segments, so we don't need to
-      // set a presentation start time.  We will assert later that this is
-      // working as expected.
-
-      // The HLS spec (RFC 8216) states in 6.3.3:
-      //
-      // "The client SHALL choose which Media Segment to play first ... the
-      // client SHOULD NOT choose a segment that starts less than three target
-      // durations from the end of the Playlist file.  Doing so can trigger
-      // playback stalls."
-      //
-      // We accomplish this in our DASH-y model by setting a presentation
-      // delay of configured value, or 3 segments duration if not configured.
-      // This will be the "live edge" of the presentation.
-      let presentationDelay;
-      if (this.config_.defaultPresentationDelay) {
-        presentationDelay = this.config_.defaultPresentationDelay;
-      } else if (this.lowLatencyPresentationDelay_) {
-        presentationDelay = this.lowLatencyPresentationDelay_;
-      } else {
-        presentationDelay = this.maxTargetDuration_ * 3;
-      }
-
-      this.presentationTimeline_ = new shaka.media.PresentationTimeline(
-      /* presentationStartTime= */ 0, /* delay= */ presentationDelay);
-      this.presentationTimeline_.setStatic(false);
+  changePresentationTimelineToLive_() {
+    // The live edge will be calculated from segments, so we don't need to
+    // set a presentation start time.  We will assert later that this is
+    // working as expected.
+
+    // The HLS spec (RFC 8216) states in 6.3.3:
+    //
+    // "The client SHALL choose which Media Segment to play first ... the
+    // client SHOULD NOT choose a segment that starts less than three target
+    // durations from the end of the Playlist file.  Doing so can trigger
+    // playback stalls."
+    //
+    // We accomplish this in our DASH-y model by setting a presentation
+    // delay of configured value, or 3 segments duration if not configured.
+    // This will be the "live edge" of the presentation.
+    let presentationDelay;
+    if (this.config_.defaultPresentationDelay) {
+      presentationDelay = this.config_.defaultPresentationDelay;
+    } else if (this.lowLatencyPresentationDelay_) {
+      presentationDelay = this.lowLatencyPresentationDelay_;
     } else {
-      this.presentationTimeline_ = new shaka.media.PresentationTimeline(
-      /* presentationStartTime= */ null, /* delay= */ 0);
-      this.presentationTimeline_.setStatic(true);
+      presentationDelay = this.maxTargetDuration_ * 3;
     }
+
+    this.presentationTimeline_.setPresentationStartTime(0);
+    this.presentationTimeline_.setDelay(presentationDelay);
+    this.presentationTimeline_.setStatic(false);
   }
 
   /**
@@ -2193,19 +2356,6 @@ shaka.hls.HlsParser = class {
     return [startByte, endByte];
   }
 
-  /** @private */
-  notifySegments_() {
-    // The presentation timeline may or may not be set yet.
-    // If it does not yet exist, hold onto the segments until it does.
-    if (!this.presentationTimeline_) {
-      return;
-    }
-    for (const segments of this.segmentsToNotifyByStream_) {
-      this.presentationTimeline_.notifySegments(segments);
-    }
-    this.segmentsToNotifyByStream_ = [];
-  }
-
   /**
    * Parses shaka.hls.Segment objects into shaka.media.SegmentReferences.
    *
@@ -2215,12 +2365,11 @@ shaka.hls.HlsParser = class {
    * @param {string} mimeType
    * @param {!Map.<number, number>} mediaSequenceToStartTime
    * @param {!Map.<string, string>} variables
-   * @param {string} codecs
    * @return {!Array.<!shaka.media.SegmentReference>}
    * @private
    */
   createSegments_(verbatimMediaPlaylistUri, playlist, type, mimeType,
-      mediaSequenceToStartTime, variables, codecs) {
+      mediaSequenceToStartTime, variables) {
     /** @type {Array.<!shaka.hls.Segment>} */
     const hlsSegments = playlist.segments;
     goog.asserts.assert(hlsSegments.length, 'Playlist should have segments!');
@@ -2423,6 +2572,57 @@ shaka.hls.HlsParser = class {
    *
    * @param {string} contentType
    * @param {string} codecs
+   * @return {?string}
+   * @private
+   */
+  guessMimeTypeBeforeLoading_(contentType, codecs) {
+    if (contentType == shaka.util.ManifestParserUtils.ContentType.TEXT) {
+      if (codecs == 'vtt' || codecs == 'wvtt') {
+        // If codecs is 'vtt', it's WebVTT.
+        return 'text/vtt';
+      } else if (codecs && codecs !== '') {
+        // Otherwise, assume MP4-embedded text, since text-based formats tend
+        // not to have a codecs string at all.
+        return 'application/mp4';
+      }
+    }
+
+    if (contentType == shaka.util.ManifestParserUtils.ContentType.IMAGE) {
+      if (!codecs || codecs == 'jpeg') {
+        return 'image/jpeg';
+      }
+    }
+
+    // Not enough information to guess from the content type and codecs.
+    return null;
+  }
+
+  /**
+   * Get a fallback mime type for the content. Used if all the better methods
+   * for determining the mime type have failed.
+   *
+   * @param {string} contentType
+   * @return {string}
+   * @private
+   */
+  guessMimeTypeFallback_(contentType) {
+    if (contentType == shaka.util.ManifestParserUtils.ContentType.TEXT) {
+      // If there was no codecs string and no content-type, assume HLS text
+      // streams are WebVTT.
+      return 'text/vtt';
+    }
+    // If the HLS content is lacking in both MIME type metadata and
+    // segment file extensions, we fall back to assuming it's MP4.
+    const map = shaka.hls.HlsParser.EXTENSION_MAP_BY_CONTENT_TYPE_[contentType];
+    return map['mp4'];
+  }
+
+  /**
+   * Attempts to guess stream's mime type based on content type, URI, and
+   * contents of the playlist.
+   *
+   * @param {string} contentType
+   * @param {string} codecs
    * @param {!shaka.hls.Playlist} playlist
    * @param {!Map.<string, string>} variables
    * @return {!Promise.<string>}
@@ -2430,7 +2630,6 @@ shaka.hls.HlsParser = class {
    */
   async guessMimeType_(contentType, codecs, playlist, variables) {
     const HlsParser = shaka.hls.HlsParser;
-    const ContentType = shaka.util.ManifestParserUtils.ContentType;
     const requestType = shaka.net.NetworkingEngine.RequestType.SEGMENT;
 
     // If you wait long enough, requesting the first segment can fail
@@ -2446,27 +2645,15 @@ shaka.hls.HlsParser = class {
     const extension = parsedUri.getPath().split('.').pop();
     const map = HlsParser.EXTENSION_MAP_BY_CONTENT_TYPE_[contentType];
 
-    const mimeType = map[extension];
+    let mimeType = map[extension];
     if (mimeType) {
       return mimeType;
     }
 
-    if (contentType == ContentType.TEXT) {
-      // The extension map didn't work.
-      if (codecs == 'vtt' || codecs == 'wvtt') {
-        // If codecs is 'vtt', it's WebVTT.
-        return 'text/vtt';
-      } else if (codecs && codecs !== '') {
-        // Otherwise, assume MP4-embedded text, since text-based formats tend
-        // not to have a codecs string at all.
-        return 'application/mp4';
-      }
-    }
-
-    if (contentType == ContentType.IMAGE) {
-      if (!codecs || codecs == 'jpeg') {
-        return 'image/jpeg';
-      }
+    // The extension map didn't work, so guess based on codecs.
+    mimeType = this.guessMimeTypeBeforeLoading_(contentType, codecs);
+    if (mimeType) {
+      return mimeType;
     }
 
     // If unable to guess mime type, request a segment and try getting it
@@ -2479,21 +2666,12 @@ shaka.hls.HlsParser = class {
         headRequest, requestType);
 
     const contentMimeType = response.headers['content-type'];
-
-    if (!contentMimeType) {
-      if (contentType == ContentType.TEXT) {
-        // If there was no codecs string and no content-type, assume HLS text
-        // streams are WebVTT.
-        return 'text/vtt';
-      }
-      // If the HLS content is lacking in both MIME type metadata and
-      // segment file extensions, we fall back to assuming it's MP4.
-      const fallbackMimeType = map['mp4'];
-      return fallbackMimeType;
+    if (contentMimeType) {
+      // Split the MIME type in case the server sent additional parameters.
+      return contentMimeType.split(';')[0];
     }
 
-    // Split the MIME type in case the server sent additional parameters.
-    return contentMimeType.split(';')[0];
+    return this.guessMimeTypeFallback_(contentType);
   }
 
   /**
@@ -2578,7 +2756,11 @@ shaka.hls.HlsParser = class {
       await this.update();
 
       // This may have converted to VOD, in which case we stop updating.
-      if (this.isLive_()) {
+      // Note that all manifests start out listed as VOD, since we won't know if
+      // a manifest is live or not until we parse at least one media playlist.
+      // So we should also keep the update timer going until we at least know
+      // if the manifest is actually VOD or not.
+      if (this.isLive_() || !this.hasEnoughInfoToFinalizeStreams_()) {
         const delay = this.updatePlaylistDelay_;
         this.updatePlaylistTimer_.tickAfter(/* seconds= */ delay);
       }
@@ -2795,12 +2977,13 @@ shaka.hls.HlsParser = class {
 /**
  * @typedef {{
  *   stream: !shaka.extern.Stream,
+ *   type: string,
  *   verbatimMediaPlaylistUri: string,
  *   absoluteMediaPlaylistUri: string,
  *   maxTimestamp: number,
- *   mediaSequenceToStartTime: !Map.<number, number>,
  *   canSkipSegments: boolean,
- *   hasEndList: boolean
+ *   hasEndList: boolean,
+ *   firstSequenceNumber: number
  * }}
  *
  * @description
@@ -2808,6 +2991,8 @@ shaka.hls.HlsParser = class {
  *
  * @property {!shaka.extern.Stream} stream
  *   The Stream itself.
+ * @property {string} type
+ *   The type value. Could be 'video', 'audio', 'text', or 'image'.
  * @property {string} verbatimMediaPlaylistUri
  *   The verbatim media playlist URI, as it appeared in the master playlist.
  *   This has not been canonicalized into an absolute URI.  This gives us a
@@ -2818,13 +3003,13 @@ shaka.hls.HlsParser = class {
  *   and updated to reflect any redirects.
  * @property {number} maxTimestamp
  *   The maximum timestamp found in the stream.
- * @property {!Map.<number, number>} mediaSequenceToStartTime
- *   A map of media sequence numbers to media start times.
  * @property {boolean} canSkipSegments
  *  True if the server supports delta playlist updates, and we can send a
  *  request for a playlist that can skip older media segments.
  * @property {boolean} hasEndList
  *  True if the stream has an EXT-X-ENDLIST tag.
+ * @property {number} firstSequenceNumber
+ *  The sequence number of the first reference. Only calculated if needed.
  */
 shaka.hls.HlsParser.StreamInfo;
 
diff --git a/lib/media/presentation_timeline.js b/lib/media/presentation_timeline.js
index 0a18b41405..d1b4d0511c 100644
--- a/lib/media/presentation_timeline.js
+++ b/lib/media/presentation_timeline.js
@@ -118,6 +118,20 @@ shaka.media.PresentationTimeline = class {
   }
 
 
+  /**
+   * Sets the presentation's start time.
+   *
+   * @param {number} presentationStartTime The wall-clock time, in seconds,
+   *   when the presentation started or will start. Only required for live.
+   * @export
+   */
+  setPresentationStartTime(presentationStartTime) {
+    goog.asserts.assert(presentationStartTime >= 0,
+        'presentationStartTime must be >= 0');
+    this.presentationStartTime_ = presentationStartTime;
+  }
+
+
   /**
    * Sets the presentation's duration.
    *
diff --git a/lib/player.js b/lib/player.js
index c9a8eeebfc..cf1c3d46da 100644
--- a/lib/player.js
+++ b/lib/player.js
@@ -1900,6 +1900,21 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     await this.filterManifest_(this.manifest_);
   }
 
+  /**
+   * @param {?shaka.extern.Variant} variant
+   * @return {!Promise}
+   * @private
+   */
+  static async loadVariant_(variant) {
+    const createSegmentIndexPromises = [];
+    for (const stream of [variant.video, variant.audio]) {
+      if (stream) {
+        createSegmentIndexPromises.push(stream.createSegmentIndex());
+      }
+    }
+    await Promise.all(createSegmentIndexPromises);
+  }
+
   /**
    * This should only be called by the load graph when it is time to load all
    * playback components needed for playback. The only times this may be called
@@ -2001,6 +2016,21 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     });
     this.abrManager_.setMediaElement(mediaElement);
 
+    // Pick the initial streams to play.
+    // Unless the user has already picked a variant, anyway, by calling
+    // selectVariantTrack before this loading stage.
+    let initialVariant = null;
+    const activeVariantTrack = this.getVariantTracks().find((t) => t.active);
+    if (!activeVariantTrack) {
+      initialVariant = this.chooseVariant_();
+      goog.asserts.assert(initialVariant, 'Must choose an initial variant!');
+    }
+
+    // Create the segment index for the chosen variant now. In case the
+    // manifest has lazy-loaded streams, this will let the manifest get enough
+    // information to determine things like duration.
+    await shaka.Player.loadVariant_(initialVariant);
+
     this.playhead_ = this.createPlayhead(has.startTime);
     this.playheadObservers_ = this.createPlayheadObserversForMSE_();
 
@@ -2047,21 +2077,34 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
           });
     }
 
-    // The event must be fired after we filter by restrictions but before the
-    // active stream is picked to allow those listening for the "streaming"
-    // event to make changes before streaming starts.
+    // Give the user a chance to change things and filter again, so they can
+    // influence what the initial variant is.
     this.dispatchEvent(
         this.makeEvent_(shaka.util.FakeEvent.EventName.Streaming));
-
-    // Pick the initial streams to play.
-    // however, we would skip switch to initial variant
-    // if user already pick variant track (via selectVariantTrack api)
-    let initialVariant = null;
-    const activeVariantTrack = this.getVariantTracks().find((t) => t.active);
-
     if (!activeVariantTrack) {
+      const oldVariant = initialVariant;
       initialVariant = this.chooseVariant_();
       goog.asserts.assert(initialVariant, 'Must choose an initial variant!');
+      if (oldVariant != initialVariant) {
+        // The user did something in the streaming event that changes the
+        // variant we would have picked, so lazy-load that variant too.
+        await shaka.Player.loadVariant_(initialVariant);
+        // Also unload the segment index of the previous initial variant, so as
+        // to not waste memory on it.
+        for (const stream of [oldVariant.video, oldVariant.audio]) {
+          if (stream && stream.closeSegmentIndex) {
+            stream.closeSegmentIndex();
+          }
+        }
+      }
+    }
+
+    // If we had to pick an initial variant before, switch to it now that the
+    // streaming engine exists.
+    if (!activeVariantTrack) {
+      goog.asserts.assert(initialVariant,
+          'Must have choosen an initial variant!');
+
       this.switchVariant_(initialVariant, /* fromAdaptation= */ true,
           /* clearBuffer= */ false, /* safeMargin= */ 0);
 
diff --git a/test/hls/hls_live_unit.js b/test/hls/hls_live_unit.js
index a19dbbae09..18ab0919ed 100644
--- a/test/hls/hls_live_unit.js
+++ b/test/hls/hls_live_unit.js
@@ -100,18 +100,16 @@ describe('HlsParser live', () => {
 
   /**
    * @param {string} master
-   * @param {string} initialMedia
-   * @param {Array=} initialReferences
-   * @return {!Promise.<shaka.extern.Manifest>}
+   * @param {string} media1
+   * @param {string} media2
    */
-  async function testInitialManifest(
-      master, initialMedia, initialReferences=null) {
+  function configureNetEngineForInitialManifest(master, media1, media2) {
     fakeNetEngine
         .setResponseText('test:/master', master)
-        .setResponseText('test:/video', initialMedia)
-        .setResponseText('test:/redirected/video', initialMedia)
-        .setResponseText('test:/video2', initialMedia)
-        .setResponseText('test:/audio', initialMedia)
+        .setResponseText('test:/video', media1)
+        .setResponseText('test:/redirected/video', media1)
+        .setResponseText('test:/video2', media2)
+        .setResponseText('test:/audio', media1)
         .setResponseValue('test:/init.mp4', initSegmentData)
         .setResponseValue('test:/main.mp4', segmentData)
         .setResponseValue('test:/main2.mp4', segmentData)
@@ -120,24 +118,35 @@ describe('HlsParser live', () => {
         .setResponseValue('test:/partial.mp4', segmentData)
         .setResponseValue('test:/partial2.mp4', segmentData)
         .setResponseValue('test:/selfInit.mp4', selfInitializingSegmentData);
+  }
+
+  /**
+   * @param {string} master
+   * @param {string} initialMedia
+   * @param {Array=} initialReferences
+   * @return {!Promise.<shaka.extern.Manifest>}
+   */
+  async function testInitialManifest(
+      master, initialMedia, initialReferences=null) {
+    configureNetEngineForInitialManifest(master, initialMedia, initialMedia);
 
     const manifest = await parser.start('test:/master', playerInterface);
 
-    if (initialReferences) {
-      await Promise.all(manifest.variants.map(async (variant) => {
-        await variant.video.createSegmentIndex();
+    // Create the segment index for the variants, to finish the lazy-loading.
+    await Promise.all(manifest.variants.map(async (variant) => {
+      await variant.video.createSegmentIndex();
 
-        // The compiler doesn't count null checks done outside this callback,
-        // so we need an assertion here.
-        goog.asserts.assert(initialReferences != null, 'references non-null');
+      if (initialReferences) {
         ManifestParser.verifySegmentIndex(variant.video, initialReferences);
+      }
 
-        if (variant.audio) {
-          await variant.audio.createSegmentIndex();
+      if (variant.audio) {
+        await variant.audio.createSegmentIndex();
+        if (initialReferences) {
           ManifestParser.verifySegmentIndex(variant.audio, initialReferences);
         }
-      }));
-    }
+      }
+    }));
 
     return manifest;
   }
@@ -415,6 +424,26 @@ describe('HlsParser live', () => {
       'main2.mp4\n',
     ].join('');
 
+    const mediaWithAdditionalSegment2 = [
+      '#EXTM3U\n',
+      '#EXT-X-TARGETDURATION:5\n',
+      '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n',
+      '#EXT-X-MEDIA-SEQUENCE:0\n',
+      '#EXTINF:2,\n',
+      'main3.mp4\n',
+      '#EXTINF:2,\n',
+      'main4.mp4\n',
+    ].join('');
+
+    const mediaWithRemovedSegment2 = [
+      '#EXTM3U\n',
+      '#EXT-X-TARGETDURATION:5\n',
+      '#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n',
+      '#EXT-X-MEDIA-SEQUENCE:1\n',
+      '#EXTINF:2,\n',
+      'main4.mp4\n',
+    ].join('');
+
     let mediaWithManySegments = [
       '#EXTM3U\n',
       '#EXT-X-TARGETDURATION:5\n',
@@ -668,6 +697,45 @@ describe('HlsParser live', () => {
         await testUpdate(manifest, mediaWithRemovedSegment, [ref2]);
       });
 
+      it('has correct references if switching after update', async () => {
+        const ref1 = makeReference(
+            'test:/main.mp4', 0, 2, /* syncTime= */ null);
+        const ref2 = makeReference(
+            'test:/main2.mp4', 2, 4, /* syncTime= */ null);
+        const ref4 = makeReference(
+            'test:/main4.mp4', 2, 4, /* syncTime= */ null);
+
+        const secondVariant = [
+          '#EXT-X-STREAM-INF:BANDWIDTH=300,CODECS="avc1",',
+          'RESOLUTION=1200x940,FRAME-RATE=60\n',
+          'video2',
+        ].join('');
+        const masterWithTwoVariants = master + secondVariant;
+        configureNetEngineForInitialManifest(masterWithTwoVariants,
+            mediaWithAdditionalSegment, mediaWithAdditionalSegment2);
+
+        const manifest = await parser.start('test:/master', playerInterface);
+        await manifest.variants[0].video.createSegmentIndex();
+        ManifestParser.verifySegmentIndex(
+            manifest.variants[0].video, [ref1, ref2]);
+        expect(manifest.variants[1].video.segmentIndex).toBeNull();
+
+        // Update.
+        fakeNetEngine
+            .setResponseText('test:/video', mediaWithRemovedSegment)
+            .setResponseText('test:/video2', mediaWithRemovedSegment2);
+        await delayForUpdatePeriod();
+
+        // Switch.
+        await manifest.variants[0].video.closeSegmentIndex();
+        await manifest.variants[1].video.createSegmentIndex();
+
+        // Check for variants to be as expected.
+        expect(manifest.variants[0].video.segmentIndex).toBeNull();
+        ManifestParser.verifySegmentIndex(
+            manifest.variants[1].video, [ref4]);
+      });
+
       it('handles updates with redirects', async () => {
         const oldRef1 = makeReference(
             'test:/main.mp4', 0, 2, /* syncTime= */ null);
diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js
index 2d6ccca6d4..97a3b7c658 100644
--- a/test/hls/hls_parser_unit.js
+++ b/test/hls/hls_parser_unit.js
@@ -129,10 +129,30 @@ describe('HlsParser', () => {
         .setResponseValue('test:/selfInit.mp4', selfInitializingSegmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
     return actual;
   }
 
+  /** @param {!shaka.extern.Manifest} manifest */
+  async function loadAllStreamsFor(manifest) {
+    const promises = [];
+    for (const variant of manifest.variants) {
+      for (const stream of [variant.video, variant.audio]) {
+        if (stream) {
+          promises.push(stream.createSegmentIndex());
+        }
+      }
+    }
+    for (const text of manifest.textStreams) {
+      promises.push(text.createSegmentIndex());
+    }
+    for (const image of manifest.imageStreams) {
+      promises.push(image.createSegmentIndex());
+    }
+    await Promise.all(promises);
+  }
+
   it('parses manifest attributes', async () => {
     const master = [
       '#EXTM3U\n',
@@ -205,6 +225,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -280,6 +301,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -321,6 +343,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -1129,6 +1152,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -1200,6 +1224,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -1263,6 +1288,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -1324,6 +1350,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -1387,6 +1414,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -1442,6 +1470,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     // Duration should be the minimum of the streams, but ignore the text
     // stream.
     const timeline = actual.presentationTimeline;
@@ -1524,6 +1553,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
 
     expect(actual.imageStreams.length).toBe(1);
     expect(actual.textStreams.length).toBe(1);
@@ -1596,6 +1626,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
 
     expect(actual.variants.length).toBe(1);
 
@@ -1721,6 +1752,7 @@ describe('HlsParser', () => {
     parser.configure(config);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     const variant = actual.variants[0];
     expect(variant.audio).toBe(null);
     expect(variant.video).toBeTruthy();
@@ -1796,6 +1828,7 @@ describe('HlsParser', () => {
     parser.configure(config);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     const variant = actual.variants[0];
     expect(variant.audio).toBeTruthy();
     expect(variant.video).toBe(null);
@@ -1871,6 +1904,7 @@ describe('HlsParser', () => {
     parser.configure(config);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     const stream = actual.textStreams[0];
     expect(stream).toBeUndefined();
   });
@@ -1945,6 +1979,7 @@ describe('HlsParser', () => {
     parser.configure(config);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     const stream = actual.imageStreams[0];
     expect(stream).toBeUndefined();
   });
@@ -1989,6 +2024,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -2040,6 +2076,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -2082,6 +2119,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -2148,6 +2186,7 @@ describe('HlsParser', () => {
           .setResponseValue('test:/main.mp4', segmentData);
 
       const actual = await parser.start('test:/master', playerInterface);
+      await loadAllStreamsFor(actual);
       expect(actual).toEqual(manifest);
     }
 
@@ -2293,6 +2332,7 @@ describe('HlsParser', () => {
 
     config.hls.ignoreTextStreamFailures = true;
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -2331,6 +2371,7 @@ describe('HlsParser', () => {
 
     config.hls.ignoreImageStreamFailures = true;
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -2396,6 +2437,7 @@ describe('HlsParser', () => {
 
     const actual =
         await parser.start('test:/host/master.m3u8', playerInterface);
+    await loadAllStreamsFor(actual);
     const video = actual.variants[0].video;
     const audio = actual.variants[0].audio;
 
@@ -2594,6 +2636,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/selfInit.mp4', selfInitializingSegmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual).toEqual(manifest);
   });
 
@@ -2638,7 +2681,8 @@ describe('HlsParser', () => {
           shaka.util.Error.Severity.CRITICAL,
           shaka.util.Error.Category.MANIFEST,
           shaka.util.Error.Code.NO_WEB_CRYPTO_API));
-      await expectAsync(parser.start('test:/master', playerInterface))
+      const actual = await parser.start('test:/master', playerInterface);
+      await expectAsync(loadAllStreamsFor(actual))
           .toBeRejectedWith(expectedError);
     } finally {
       Object.defineProperty(window, 'crypto', {
@@ -2883,8 +2927,9 @@ describe('HlsParser', () => {
      * @param {string} master
      * @param {string} media
      * @param {!shaka.util.Error} error
+     * @param {boolean=} onCreateSegmentIndex
      */
-    async function verifyError(master, media, error) {
+    async function verifyError(master, media, error, onCreateSegmentIndex) {
       fakeNetEngine
           .setResponseText('test:/master', master)
           .setResponseText('test:/audio', media)
@@ -2895,8 +2940,14 @@ describe('HlsParser', () => {
           .setResponseValue('data:text/plain;base64,AAECAwQFBgcICQoLDA0ODw==',
               aes128Key);
 
-      await expectAsync(parser.start('test:/master', playerInterface))
-          .toBeRejectedWith(Util.jasmineError(error));
+      if (onCreateSegmentIndex) {
+        const actual = await parser.start('test:/master', playerInterface);
+        await expectAsync(loadAllStreamsFor(actual))
+            .toBeRejectedWith(Util.jasmineError(error));
+      } else {
+        await expectAsync(parser.start('test:/master', playerInterface))
+            .toBeRejectedWith(Util.jasmineError(error));
+      }
     }
 
     it('if unable to guess codecs', async () => {
@@ -3033,7 +3084,7 @@ describe('HlsParser', () => {
           shaka.util.Error.Category.MANIFEST,
           Code.HLS_MSE_ENCRYPTED_MP2T_NOT_SUPPORTED);
 
-      await verifyError(master, media, error);
+      await verifyError(master, media, error, /* onCreateSegmentIndex= */ true);
     });
 
     describe('if required tags are missing', () => {
@@ -3049,7 +3100,8 @@ describe('HlsParser', () => {
             Code.HLS_REQUIRED_TAG_MISSING,
             tagName);
 
-        await verifyError(master, media, error);
+        await verifyError(
+            master, media, error, /* onCreateSegmentIndex= */ true);
       }
 
       it('EXTINF', async () => {
@@ -3383,6 +3435,7 @@ describe('HlsParser', () => {
         .setResponseValue('test:/main.mp4', segmentData);
 
     const actual = await parser.start('test:/master', playerInterface);
+    await loadAllStreamsFor(actual);
     expect(actual.variants.length).toBe(1);
     expect(actual).toEqual(manifest);
   });
@@ -3415,6 +3468,7 @@ describe('HlsParser', () => {
 
       const actual =
           await parser.start('test:/host/master.m3u8', playerInterface);
+      await loadAllStreamsFor(actual);
       const video = actual.variants[0].video;
       const audio = actual.variants[0].audio;
 
@@ -3462,6 +3516,7 @@ describe('HlsParser', () => {
 
       const actual =
           await parser.start('test:/host/master.m3u8', playerInterface);
+      await loadAllStreamsFor(actual);
       const video = actual.variants[0].video;
       const audio = actual.variants[0].audio;
 
@@ -3509,6 +3564,7 @@ describe('HlsParser', () => {
 
       const actual =
           await parser.start('test:/host/master.m3u8', playerInterface);
+      await loadAllStreamsFor(actual);
       const video = actual.variants[0].video;
       const audio = actual.variants[0].audio;