Skip to content

Commit

Permalink
Merge pull request #3993 from video-dev/feature/sync-live-subtitle-tr…
Browse files Browse the repository at this point in the history 
…acks-to-main-msn

Align live WebVTT subtitle playlists with main playlist
  • Loading branch information
@robwalch
robwalch committed Jun 4, 2021
2 parents 9d9fc59 + 2e3c3ef commit 316b99d
Show file tree
Hide file tree
Showing 9 changed files with 157 additions and 95 deletions.
2 changes: 2 additions & 0 deletions api-extractor/report/hls.js.api.md
View file
Expand Up @@ -212,6 +212,8 @@ export interface BufferFlushingData {
// (undocumented)
endOffset: number;
// (undocumented)
endOffsetSubtitles?: number;
// (undocumented)
startOffset: number;
// (undocumented)
type: SourceBufferName | null;
Expand Down
36 changes: 10 additions & 26 deletions src/controller/audio-stream-controller.ts
View file
@@ -1,20 +1,20 @@
import BaseStreamController, { State } from './base-stream-controller';
import type { NetworkComponentAPI } from '../types/component-api';
import { Events } from '../events';
import { BufferHelper } from '../utils/buffer-helper';
import type { FragmentTracker } from './fragment-tracker';
import { FragmentState } from './fragment-tracker';
import { Level } from '../types/level';
import { PlaylistLevelType } from '../types/loader';
import { Fragment, ElementaryStreamTypes, Part } from '../loader/fragment';
import ChunkCache from '../demux/chunk-cache';
import TransmuxerInterface from '../demux/transmuxer-interface';
import type { TransmuxerResult } from '../types/transmuxer';
import { ChunkMetadata } from '../types/transmuxer';
import { fragmentWithinToleranceTest } from './fragment-finders';
import { alignPDT } from '../utils/discontinuities';
import { ErrorDetails } from '../errors';
import { logger } from '../utils/logger';
import type { NetworkComponentAPI } from '../types/component-api';
import type { FragmentTracker } from './fragment-tracker';
import type { TransmuxerResult } from '../types/transmuxer';
import type Hls from '../hls';
import type { LevelDetails } from '../loader/level-details';
import type { TrackSet } from '../types/track';
Expand All @@ -31,8 +31,8 @@ import type {
FragParsingMetadataData,
FragParsingUserdataData,
FragBufferedData,
ErrorData,
} from '../types/events';
import type { ErrorData } from '../types/events';

const TICK_INTERVAL = 100; // how often to tick in ms

Expand Down Expand Up @@ -411,24 +411,7 @@ class AudioStreamController
}

onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) {
if (this.mainDetails === null) {
const mainDetails = (this.mainDetails = data.details);
// compute start position if we haven't already
const trackId = this.levelLastLoaded;
if (
trackId !== null &&
this.levels &&
this.startPosition === -1 &&
mainDetails.live
) {
const track = this.levels[trackId];
if (!track.details || !track.details.fragments[0]) {
return;
}
alignPDT(track.details, mainDetails);
this.setStartPosition(track.details, track.details.fragments[0].start);
}
}
this.mainDetails = data.details;
}

onAudioTrackLoaded(event: Events.AUDIO_TRACK_LOADED, data: TrackLoadedData) {
Expand All @@ -445,18 +428,19 @@ class AudioStreamController
const track = levels[trackId];
let sliding = 0;
if (newDetails.live || track.details?.live) {
const mainDetails = this.mainDetails;
if (!newDetails.fragments[0]) {
newDetails.deltaUpdateFailed = true;
}
if (newDetails.deltaUpdateFailed) {
if (newDetails.deltaUpdateFailed || !mainDetails) {
return;
}
if (
!track.details &&
this.mainDetails?.hasProgramDateTime &&
newDetails.hasProgramDateTime
newDetails.hasProgramDateTime &&
mainDetails.hasProgramDateTime
) {
alignPDT(newDetails, this.mainDetails);
alignPDT(newDetails, mainDetails);
sliding = newDetails.fragments[0].start;
} else {
sliding = this.alignPlaylists(newDetails, track.details);
Expand Down
17 changes: 10 additions & 7 deletions src/controller/level-helper.ts
View file
Expand Up @@ -372,17 +372,20 @@ export function adjustSliding(
const delta =
newDetails.startSN + newDetails.skippedSegments - oldDetails.startSN;
const oldFragments = oldDetails.fragments;
const newFragments = newDetails.fragments;
if (delta < 0 || delta >= oldFragments.length) {
return;
}
const playlistStartOffset = oldFragments[delta].start;
if (playlistStartOffset) {
for (let i = newDetails.skippedSegments; i < newFragments.length; i++) {
newFragments[i].start += playlistStartOffset;
addSliding(newDetails, oldFragments[delta].start);
}

export function addSliding(details: LevelDetails, start: number) {
if (start) {
const fragments = details.fragments;
for (let i = details.skippedSegments; i < fragments.length; i++) {
fragments[i].start += start;
}
if (newDetails.fragmentHint) {
newDetails.fragmentHint.start += playlistStartOffset;
if (details.fragmentHint) {
details.fragmentHint.start += start;
}
}
}
Expand Down
131 changes: 96 additions & 35 deletions src/controller/subtitle-stream-controller.ts
View file
Expand Up @@ -2,11 +2,13 @@ import { Events } from '../events';
import { logger } from '../utils/logger';
import { BufferHelper } from '../utils/buffer-helper';
import { findFragmentByPDT, findFragmentByPTS } from './fragment-finders';
import type { FragmentTracker } from './fragment-tracker';
import { alignPDT } from '../utils/discontinuities';
import { addSliding } from './level-helper';
import { FragmentState } from './fragment-tracker';
import BaseStreamController, { State } from './base-stream-controller';
import { PlaylistLevelType } from '../types/loader';
import { Level } from '../types/level';
import type { FragmentTracker } from './fragment-tracker';
import type { NetworkComponentAPI } from '../types/component-api';
import type Hls from '../hls';
import type { LevelDetails } from '../loader/level-details';
Expand All @@ -19,6 +21,7 @@ import type {
TrackLoadedData,
TrackSwitchedData,
BufferFlushingData,
LevelLoadedData,
} from '../types/events';

const TICK_INTERVAL = 500; // how often to tick in ms
Expand All @@ -36,16 +39,24 @@ export class SubtitleStreamController

private currentTrackId: number = -1;
private tracksBuffered: Array<TimeRange[]> = [];
private mainDetails: LevelDetails | null = null;

constructor(hls: Hls, fragmentTracker: FragmentTracker) {
super(hls, fragmentTracker, '[subtitle-stream-controller]');
this._registerListeners();
}

protected onHandlerDestroying() {
this._unregisterListeners();
this.mainDetails = null;
}

private _registerListeners() {
const { hls } = this;
hls.on(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
hls.on(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
hls.on(Events.MANIFEST_LOADING, this.onManifestLoading, this);
hls.on(Events.LEVEL_LOADED, this.onLevelLoaded, this);
hls.on(Events.ERROR, this.onError, this);
hls.on(Events.SUBTITLE_TRACKS_UPDATED, this.onSubtitleTracksUpdated, this);
hls.on(Events.SUBTITLE_TRACK_SWITCH, this.onSubtitleTrackSwitch, this);
Expand All @@ -58,6 +69,8 @@ export class SubtitleStreamController
const { hls } = this;
hls.off(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
hls.off(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
hls.off(Events.MANIFEST_LOADING, this.onManifestLoading, this);
hls.off(Events.LEVEL_LOADED, this.onLevelLoaded, this);
hls.off(Events.ERROR, this.onError, this);
hls.off(Events.SUBTITLE_TRACKS_UPDATED, this.onSubtitleTracksUpdated, this);
hls.off(Events.SUBTITLE_TRACK_SWITCH, this.onSubtitleTrackSwitch, this);
Expand All @@ -70,17 +83,17 @@ export class SubtitleStreamController
this.stopLoad();
this.state = State.IDLE;

// Check if we already have a track with necessary details to load fragments
const currentTrack = this.levels[this.currentTrackId];
if (currentTrack?.details) {
this.setInterval(TICK_INTERVAL);
this.tick();
}
this.setInterval(TICK_INTERVAL);
this.tick();
}

onHandlerDestroyed() {
this._unregisterListeners();
super.onHandlerDestroyed();
onManifestLoading() {
this.mainDetails = null;
this.fragmentTracker.removeAllFragments();
}

onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) {
this.mainDetails = data.details;
}

onSubtitleFragProcessed(
Expand Down Expand Up @@ -123,18 +136,31 @@ export class SubtitleStreamController
this.fragmentTracker.fragBuffered(frag);
}

onBufferFlushing(
event: Events.BUFFER_FLUSHING,
{ startOffset, endOffset }: BufferFlushingData
) {
onBufferFlushing(event: Events.BUFFER_FLUSHING, data: BufferFlushingData) {
const { startOffset, endOffset } = data;
if (startOffset === 0 && endOffset !== Number.POSITIVE_INFINITY) {
const { currentTrackId, levels } = this;
if (
!levels.length ||
!levels[currentTrackId] ||
!levels[currentTrackId].details
) {
return;
}
const trackDetails = levels[currentTrackId].details as LevelDetails;
const targetDuration = trackDetails.targetduration;
const endOffsetSubtitles = endOffset - targetDuration;
if (endOffsetSubtitles <= 0) {
return;
}
data.endOffsetSubtitles = Math.max(0, endOffsetSubtitles);
this.tracksBuffered.forEach((buffered) => {
for (let i = 0; i < buffered.length; ) {
if (buffered[i].end <= endOffset) {
if (buffered[i].end <= endOffsetSubtitles) {
buffered.shift();
continue;
} else if (buffered[i].start < endOffset) {
buffered[i].start = endOffset;
} else if (buffered[i].start < endOffsetSubtitles) {
buffered[i].start = endOffsetSubtitles;
} else {
break;
}
Expand All @@ -143,7 +169,7 @@ export class SubtitleStreamController
});
this.fragmentTracker.removeFragmentsInRange(
startOffset,
endOffset,
endOffsetSubtitles,
PlaylistLevelType.SUBTITLE
);
}
Expand Down Expand Up @@ -207,27 +233,61 @@ export class SubtitleStreamController
event: Events.SUBTITLE_TRACK_LOADED,
data: TrackLoadedData
) {
const { id, details } = data;
const { details: newDetails, id: trackId } = data;
const { currentTrackId, levels } = this;
if (!levels.length || !details) {
if (!levels.length) {
return;
}
const currentTrack: Level = levels[currentTrackId];
if (id >= levels.length || id !== currentTrackId || !currentTrack) {
const track: Level = levels[currentTrackId];
if (trackId >= levels.length || trackId !== currentTrackId || !track) {
return;
}
this.mediaBuffer = this.mediaBufferTimeRanges;
if (details.live || currentTrack.details?.live) {
if (details.deltaUpdateFailed) {
if (newDetails.live || track.details?.live) {
const mainDetails = this.mainDetails;
if (newDetails.deltaUpdateFailed || !mainDetails) {
return;
}
// TODO: Subtitle Fragments should be assigned startPTS and endPTS once VTT/TTML is parsed
// otherwise this depends on DISCONTINUITY or PROGRAM-DATE-TIME tags to align playlists
this.alignPlaylists(details, currentTrack.details);
const mainSlidingStartFragment = mainDetails.fragments[0];
if (!track.details) {
if (newDetails.hasProgramDateTime && mainDetails.hasProgramDateTime) {
alignPDT(newDetails, mainDetails);
} else if (mainSlidingStartFragment) {
// line up live playlist with main so that fragments in range are loaded
addSliding(newDetails, mainSlidingStartFragment.start);
}
} else {
const sliding = this.alignPlaylists(newDetails, track.details);
if (sliding === 0 && mainSlidingStartFragment) {
// realign with main when there is no overlap with last refresh
addSliding(newDetails, mainSlidingStartFragment.start);
}
}
}
track.details = newDetails;
this.levelLastLoaded = trackId;

// trigger handler right now
this.tick();

// If playlist is misaligned because of bad PDT or drift, delete details to resync with main on reload
if (
newDetails.live &&
!this.fragCurrent &&
this.media &&
this.state === State.IDLE
) {
const foundFrag = findFragmentByPTS(
null,
newDetails.fragments,
this.media.currentTime,
0
);
if (!foundFrag) {
this.warn('Subtitle playlist not aligned with playback');
track.details = undefined;
}
}
currentTrack.details = details;
this.levelLastLoaded = id;
this.setInterval(TICK_INTERVAL);
}

_handleFragmentLoadComplete(fragLoadedData: FragLoadedData) {
Expand Down Expand Up @@ -285,29 +345,30 @@ export class SubtitleStreamController
return;
}

// Expand range of subs loaded by one target-duration in either direction to make up for misaligned playlists
const trackDetails = levels[currentTrackId].details as LevelDetails;
const targetDuration = trackDetails.targetduration;
const { config, media } = this;
const bufferedInfo = BufferHelper.bufferedInfo(
this.mediaBufferTimeRanges,
media.currentTime,
media.currentTime - targetDuration,
config.maxBufferHole
);
const { end: targetBufferTime, len: bufferLen } = bufferedInfo;

const maxBufLen = this.getMaxBufferLength();
const maxBufLen = this.getMaxBufferLength() + targetDuration;

if (bufferLen > maxBufLen) {
return;
}

const trackDetails = levels[currentTrackId].details as LevelDetails;
console.assert(
trackDetails,
'Subtitle track details are defined on idle subtitle stream controller tick'
);
const fragments = trackDetails.fragments;
const fragLen = fragments.length;
const end =
fragments[fragLen - 1].start + fragments[fragLen - 1].duration;
const end = trackDetails.edge;

let foundFrag;
const fragPrevious = this.fragPrevious;
Expand Down

0 comments on commit 316b99d

Please sign in to comment.