Align live WebVTT subtitle playlists with main playlist

api-extractor/report/hls.js.api.md

@@ -212,6 +212,8 @@ export interface BufferFlushingData {
     // (undocumented)
     endOffset: number;
     // (undocumented)
+    endOffsetSubtitles?: number;
+    // (undocumented)
     startOffset: number;
     // (undocumented)
     type: SourceBufferName | null;

src/controller/audio-stream-controller.ts

@@ -1,20 +1,20 @@
 import BaseStreamController, { State } from './base-stream-controller';
-import type { NetworkComponentAPI } from '../types/component-api';
 import { Events } from '../events';
 import { BufferHelper } from '../utils/buffer-helper';
-import type { FragmentTracker } from './fragment-tracker';
 import { FragmentState } from './fragment-tracker';
 import { Level } from '../types/level';
 import { PlaylistLevelType } from '../types/loader';
 import { Fragment, ElementaryStreamTypes, Part } from '../loader/fragment';
 import ChunkCache from '../demux/chunk-cache';
 import TransmuxerInterface from '../demux/transmuxer-interface';
-import type { TransmuxerResult } from '../types/transmuxer';
 import { ChunkMetadata } from '../types/transmuxer';
 import { fragmentWithinToleranceTest } from './fragment-finders';
 import { alignPDT } from '../utils/discontinuities';
 import { ErrorDetails } from '../errors';
 import { logger } from '../utils/logger';
+import type { NetworkComponentAPI } from '../types/component-api';
+import type { FragmentTracker } from './fragment-tracker';
+import type { TransmuxerResult } from '../types/transmuxer';
 import type Hls from '../hls';
 import type { LevelDetails } from '../loader/level-details';
 import type { TrackSet } from '../types/track';
@@ -31,8 +31,8 @@ import type {
   FragParsingMetadataData,
   FragParsingUserdataData,
   FragBufferedData,
+  ErrorData,
 } from '../types/events';
-import type { ErrorData } from '../types/events';
 
 const TICK_INTERVAL = 100; // how often to tick in ms
 
@@ -411,24 +411,7 @@ class AudioStreamController
   }
 
   onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) {
-    if (this.mainDetails === null) {
-      const mainDetails = (this.mainDetails = data.details);
-      // compute start position if we haven't already
-      const trackId = this.levelLastLoaded;
-      if (
-        trackId !== null &&
-        this.levels &&
-        this.startPosition === -1 &&
-        mainDetails.live
-      ) {
-        const track = this.levels[trackId];
-        if (!track.details || !track.details.fragments[0]) {
-          return;
-        }
-        alignPDT(track.details, mainDetails);
-        this.setStartPosition(track.details, track.details.fragments[0].start);
-      }
-    }
+    this.mainDetails = data.details;
   }
 
   onAudioTrackLoaded(event: Events.AUDIO_TRACK_LOADED, data: TrackLoadedData) {
@@ -445,18 +428,19 @@ class AudioStreamController
     const track = levels[trackId];
     let sliding = 0;
     if (newDetails.live || track.details?.live) {
+      const mainDetails = this.mainDetails;
       if (!newDetails.fragments[0]) {
         newDetails.deltaUpdateFailed = true;
       }
-      if (newDetails.deltaUpdateFailed) {
+      if (newDetails.deltaUpdateFailed || !mainDetails) {
         return;
       }
       if (
         !track.details &&
-        this.mainDetails?.hasProgramDateTime &&
-        newDetails.hasProgramDateTime
+        newDetails.hasProgramDateTime &&
+        mainDetails.hasProgramDateTime
       ) {
-        alignPDT(newDetails, this.mainDetails);
+        alignPDT(newDetails, mainDetails);
         sliding = newDetails.fragments[0].start;
       } else {
         sliding = this.alignPlaylists(newDetails, track.details);

src/controller/level-helper.ts

@@ -372,17 +372,20 @@ export function adjustSliding(
   const delta =
     newDetails.startSN + newDetails.skippedSegments - oldDetails.startSN;
   const oldFragments = oldDetails.fragments;
-  const newFragments = newDetails.fragments;
   if (delta < 0 || delta >= oldFragments.length) {
     return;
   }
-  const playlistStartOffset = oldFragments[delta].start;
-  if (playlistStartOffset) {
-    for (let i = newDetails.skippedSegments; i < newFragments.length; i++) {
-      newFragments[i].start += playlistStartOffset;
+  addSliding(newDetails, oldFragments[delta].start);
+}
+
+export function addSliding(details: LevelDetails, start: number) {
+  if (start) {
+    const fragments = details.fragments;
+    for (let i = details.skippedSegments; i < fragments.length; i++) {
+      fragments[i].start += start;
     }
-    if (newDetails.fragmentHint) {
-      newDetails.fragmentHint.start += playlistStartOffset;
+    if (details.fragmentHint) {
+      details.fragmentHint.start += start;
     }
   }
 }

src/controller/subtitle-stream-controller.ts

@@ -2,11 +2,13 @@ import { Events } from '../events';
 import { logger } from '../utils/logger';
 import { BufferHelper } from '../utils/buffer-helper';
 import { findFragmentByPDT, findFragmentByPTS } from './fragment-finders';
-import type { FragmentTracker } from './fragment-tracker';
+import { alignPDT } from '../utils/discontinuities';
+import { addSliding } from './level-helper';
 import { FragmentState } from './fragment-tracker';
 import BaseStreamController, { State } from './base-stream-controller';
 import { PlaylistLevelType } from '../types/loader';
 import { Level } from '../types/level';
+import type { FragmentTracker } from './fragment-tracker';
 import type { NetworkComponentAPI } from '../types/component-api';
 import type Hls from '../hls';
 import type { LevelDetails } from '../loader/level-details';
@@ -19,6 +21,7 @@ import type {
   TrackLoadedData,
   TrackSwitchedData,
   BufferFlushingData,
+  LevelLoadedData,
 } from '../types/events';
 
 const TICK_INTERVAL = 500; // how often to tick in ms
@@ -36,16 +39,24 @@ export class SubtitleStreamController
 
   private currentTrackId: number = -1;
   private tracksBuffered: Array<TimeRange[]> = [];
+  private mainDetails: LevelDetails | null = null;
 
   constructor(hls: Hls, fragmentTracker: FragmentTracker) {
     super(hls, fragmentTracker, '[subtitle-stream-controller]');
     this._registerListeners();
   }
 
+  protected onHandlerDestroying() {
+    this._unregisterListeners();
+    this.mainDetails = null;
+  }
+
   private _registerListeners() {
     const { hls } = this;
     hls.on(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
     hls.on(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
+    hls.on(Events.MANIFEST_LOADING, this.onManifestLoading, this);
+    hls.on(Events.LEVEL_LOADED, this.onLevelLoaded, this);
     hls.on(Events.ERROR, this.onError, this);
     hls.on(Events.SUBTITLE_TRACKS_UPDATED, this.onSubtitleTracksUpdated, this);
     hls.on(Events.SUBTITLE_TRACK_SWITCH, this.onSubtitleTrackSwitch, this);
@@ -58,6 +69,8 @@ export class SubtitleStreamController
     const { hls } = this;
     hls.off(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
     hls.off(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
+    hls.off(Events.MANIFEST_LOADING, this.onManifestLoading, this);
+    hls.off(Events.LEVEL_LOADED, this.onLevelLoaded, this);
     hls.off(Events.ERROR, this.onError, this);
     hls.off(Events.SUBTITLE_TRACKS_UPDATED, this.onSubtitleTracksUpdated, this);
     hls.off(Events.SUBTITLE_TRACK_SWITCH, this.onSubtitleTrackSwitch, this);
@@ -70,17 +83,17 @@ export class SubtitleStreamController
     this.stopLoad();
     this.state = State.IDLE;
 
-    // Check if we already have a track with necessary details to load fragments
-    const currentTrack = this.levels[this.currentTrackId];
-    if (currentTrack?.details) {
-      this.setInterval(TICK_INTERVAL);
-      this.tick();
-    }
+    this.setInterval(TICK_INTERVAL);
+    this.tick();
   }
 
-  onHandlerDestroyed() {
-    this._unregisterListeners();
-    super.onHandlerDestroyed();
+  onManifestLoading() {
+    this.mainDetails = null;
+    this.fragmentTracker.removeAllFragments();
+  }
+
+  onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) {
+    this.mainDetails = data.details;
   }
 
   onSubtitleFragProcessed(
@@ -123,18 +136,31 @@ export class SubtitleStreamController
     this.fragmentTracker.fragBuffered(frag);
   }
 
-  onBufferFlushing(
-    event: Events.BUFFER_FLUSHING,
-    { startOffset, endOffset }: BufferFlushingData
-  ) {
+  onBufferFlushing(event: Events.BUFFER_FLUSHING, data: BufferFlushingData) {
+    const { startOffset, endOffset } = data;
     if (startOffset === 0 && endOffset !== Number.POSITIVE_INFINITY) {
+      const { currentTrackId, levels } = this;
+      if (
+        !levels.length ||
+        !levels[currentTrackId] ||
+        !levels[currentTrackId].details
+      ) {
+        return;
+      }
+      const trackDetails = levels[currentTrackId].details as LevelDetails;
+      const targetDuration = trackDetails.targetduration;
+      const endOffsetSubtitles = endOffset - targetDuration;
+      if (endOffsetSubtitles <= 0) {
+        return;
+      }
+      data.endOffsetSubtitles = Math.max(0, endOffsetSubtitles);
       this.tracksBuffered.forEach((buffered) => {
         for (let i = 0; i < buffered.length; ) {
-          if (buffered[i].end <= endOffset) {
+          if (buffered[i].end <= endOffsetSubtitles) {
             buffered.shift();
             continue;
-          } else if (buffered[i].start < endOffset) {
-            buffered[i].start = endOffset;
+          } else if (buffered[i].start < endOffsetSubtitles) {
+            buffered[i].start = endOffsetSubtitles;
           } else {
             break;
           }
@@ -143,7 +169,7 @@ export class SubtitleStreamController
       });
       this.fragmentTracker.removeFragmentsInRange(
         startOffset,
-        endOffset,
+        endOffsetSubtitles,
         PlaylistLevelType.SUBTITLE
       );
     }
@@ -207,27 +233,61 @@ export class SubtitleStreamController
     event: Events.SUBTITLE_TRACK_LOADED,
     data: TrackLoadedData
   ) {
-    const { id, details } = data;
+    const { details: newDetails, id: trackId } = data;
     const { currentTrackId, levels } = this;
-    if (!levels.length || !details) {
+    if (!levels.length) {
       return;
     }
-    const currentTrack: Level = levels[currentTrackId];
-    if (id >= levels.length || id !== currentTrackId || !currentTrack) {
+    const track: Level = levels[currentTrackId];
+    if (trackId >= levels.length || trackId !== currentTrackId || !track) {
       return;
     }
     this.mediaBuffer = this.mediaBufferTimeRanges;
-    if (details.live || currentTrack.details?.live) {
-      if (details.deltaUpdateFailed) {
+    if (newDetails.live || track.details?.live) {
+      const mainDetails = this.mainDetails;
+      if (newDetails.deltaUpdateFailed || !mainDetails) {
         return;
       }
-      // TODO: Subtitle Fragments should be assigned startPTS and endPTS once VTT/TTML is parsed
-      //  otherwise this depends on DISCONTINUITY or PROGRAM-DATE-TIME tags to align playlists
-      this.alignPlaylists(details, currentTrack.details);
+      const mainSlidingStartFragment = mainDetails.fragments[0];
+      if (!track.details) {
+        if (newDetails.hasProgramDateTime && mainDetails.hasProgramDateTime) {
+          alignPDT(newDetails, mainDetails);
+        } else if (mainSlidingStartFragment) {
+          // line up live playlist with main so that fragments in range are loaded
+          addSliding(newDetails, mainSlidingStartFragment.start);
+        }
+      } else {
+        const sliding = this.alignPlaylists(newDetails, track.details);
+        if (sliding === 0 && mainSlidingStartFragment) {
+          // realign with main when there is no overlap with last refresh
+          addSliding(newDetails, mainSlidingStartFragment.start);
+        }
+      }
+    }
+    track.details = newDetails;
+    this.levelLastLoaded = trackId;
+
+    // trigger handler right now
+    this.tick();
+
+    // If playlist is misaligned because of bad PDT or drift, delete details to resync with main on reload
+    if (
+      newDetails.live &&
+      !this.fragCurrent &&
+      this.media &&
+      this.state === State.IDLE
+    ) {
+      const foundFrag = findFragmentByPTS(
+        null,
+        newDetails.fragments,
+        this.media.currentTime,
+        0
+      );
+      if (!foundFrag) {
+        this.warn('Subtitle playlist not aligned with playback');
+        track.details = undefined;
+      }
     }
-    currentTrack.details = details;
-    this.levelLastLoaded = id;
-    this.setInterval(TICK_INTERVAL);
   }
 
   _handleFragmentLoadComplete(fragLoadedData: FragLoadedData) {
@@ -285,29 +345,30 @@ export class SubtitleStreamController
         return;
       }
 
+      // Expand range of subs loaded by one target-duration in either direction to make up for misaligned playlists
+      const trackDetails = levels[currentTrackId].details as LevelDetails;
+      const targetDuration = trackDetails.targetduration;
       const { config, media } = this;
       const bufferedInfo = BufferHelper.bufferedInfo(
         this.mediaBufferTimeRanges,
-        media.currentTime,
+        media.currentTime - targetDuration,
         config.maxBufferHole
       );
       const { end: targetBufferTime, len: bufferLen } = bufferedInfo;
 
-      const maxBufLen = this.getMaxBufferLength();
+      const maxBufLen = this.getMaxBufferLength() + targetDuration;
 
       if (bufferLen > maxBufLen) {
         return;
       }
 
-      const trackDetails = levels[currentTrackId].details as LevelDetails;
       console.assert(
         trackDetails,
         'Subtitle track details are defined on idle subtitle stream controller tick'
       );
       const fragments = trackDetails.fragments;
       const fragLen = fragments.length;
-      const end =
-        fragments[fragLen - 1].start + fragments[fragLen - 1].duration;
+      const end = trackDetails.edge;
 
       let foundFrag;
       const fragPrevious = this.fragPrevious;