Skip to content

Commit

Permalink
Merge pull request #5597 from video-dev/bugfix/subtitle-target-durati…
Browse files Browse the repository at this point in the history 
…on-impacts-segment-selection

Bugfix: large subtitle target duration can result in subtitles not loading as needed
  • Loading branch information
@robwalch
robwalch committed Jun 28, 2023
2 parents 30757b1 + 11d9097 commit 7002632
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 50 deletions.
41 changes: 21 additions & 20 deletions src/controller/subtitle-stream-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,17 +165,7 @@ export class SubtitleStreamController
onBufferFlushing(event: Events.BUFFER_FLUSHING, data: BufferFlushingData) {
const { startOffset, endOffset } = data;
if (startOffset === 0 && endOffset !== Number.POSITIVE_INFINITY) {
const { currentTrackId, levels } = this;
if (
!levels.length ||
!levels[currentTrackId] ||
!levels[currentTrackId].details
) {
return;
}
const trackDetails = levels[currentTrackId].details as LevelDetails;
const targetDuration = trackDetails.targetduration;
const endOffsetSubtitles = endOffset - targetDuration;
const endOffsetSubtitles = endOffset - 1;
if (endOffsetSubtitles <= 0) {
return;
}
Expand Down Expand Up @@ -408,15 +398,11 @@ export class SubtitleStreamController
if (!levels.length || !track || !track.details) {
return;
}

// Expand range of subs loaded by one target-duration in either direction to make up for misaligned playlists
const trackDetails = track.details as LevelDetails;
const targetDuration = trackDetails.targetduration;
const { config } = this;
const currentTime = this.getLoadPosition();
const bufferedInfo = BufferHelper.bufferedInfo(
this.tracksBuffered[this.currentTrackId] || [],
currentTime - targetDuration,
currentTime,
config.maxBufferHole
);
const { end: targetBufferTime, len: bufferLen } = bufferedInfo;
Expand All @@ -425,8 +411,10 @@ export class SubtitleStreamController
this.media,
PlaylistLevelType.MAIN
);
const trackDetails = track.details as LevelDetails;
const maxBufLen =
this.getMaxBufferLength(mainBufferInfo?.len) + targetDuration;
this.getMaxBufferLength(mainBufferInfo?.len) +
trackDetails.levelTargetDuration;

if (bufferLen > maxBufLen) {
return;
Expand All @@ -438,12 +426,14 @@ export class SubtitleStreamController
let foundFrag: Fragment | null = null;
const fragPrevious = this.fragPrevious;
if (targetBufferTime < end) {
const { maxFragLookUpTolerance } = config;
const tolerance = config.maxFragLookUpTolerance;
const lookupTolerance =
targetBufferTime > end - tolerance ? 0 : tolerance;
foundFrag = findFragmentByPTS(
fragPrevious,
fragments,
Math.max(fragments[0].start, targetBufferTime),
maxFragLookUpTolerance
lookupTolerance
);
if (
!foundFrag &&
Expand All @@ -458,8 +448,19 @@ export class SubtitleStreamController
if (!foundFrag) {
return;
}

foundFrag = this.mapToInitFragWhenRequired(foundFrag) as Fragment;
if (foundFrag.sn !== 'initSegment') {
// Load earlier fragment in same discontinuity to make up for misaligned playlists and cues that extend beyond end of segment
const curSNIdx = foundFrag.sn - trackDetails.startSN;
const prevFrag = fragments[curSNIdx - 1];
if (
prevFrag &&
prevFrag.cc === foundFrag.cc &&
this.fragmentTracker.getState(prevFrag) === FragmentState.NOT_LOADED
) {
foundFrag = prevFrag;
}
}
if (
this.fragmentTracker.getState(foundFrag) === FragmentState.NOT_LOADED
) {
Expand Down
47 changes: 23 additions & 24 deletions src/controller/timeline-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ export class TimelineController implements ComponentAPI {
this.captionsTracks = {};
this.nonNativeCaptionsTracks = {};
this.textTracks = [];
this.unparsedVttFrags = this.unparsedVttFrags || [];
this.unparsedVttFrags = [];
this.initPTS = [];
if (this.cea608Parser1 && this.cea608Parser2) {
this.cea608Parser1.reset();
Expand Down Expand Up @@ -477,24 +477,9 @@ export class TimelineController implements ComponentAPI {
data: FragDecryptedData | FragLoadedData
) {
const { frag, payload } = data;
const { initPTS, unparsedVttFrags } = this;
if (frag.type === PlaylistLevelType.SUBTITLE) {
// If fragment is subtitle type, parse as WebVTT.
if (payload.byteLength) {
// We need an initial synchronisation PTS. Store fragments as long as none has arrived.
if (!initPTS[frag.cc]) {
unparsedVttFrags.push(data);
if (initPTS.length) {
// finish unsuccessfully, otherwise the subtitle-stream-controller could be blocked from loading new frags.
this.hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, {
success: false,
frag,
error: new Error('Missing initial subtitle PTS'),
});
}
return;
}

const decryptData = frag.decryptdata;
// fragment after decryption has a stats object
const decrypted = 'stats' in data;
Expand All @@ -516,7 +501,7 @@ export class TimelineController implements ComponentAPI {
) {
this._parseIMSC1(frag, payload);
} else {
this._parseVTTs(frag, payload, vttCCs);
this._parseVTTs(data);
}
}
} else {
Expand Down Expand Up @@ -553,7 +538,16 @@ export class TimelineController implements ComponentAPI {
);
}

private _parseVTTs(frag: Fragment, payload: ArrayBuffer, vttCCs: any) {
private _parseVTTs(data: FragDecryptedData | FragLoadedData) {
const { frag, payload } = data;
// We need an initial synchronisation PTS. Store fragments as long as none has arrived
const { initPTS, unparsedVttFrags } = this;
const maxAvCC = initPTS.length - 1;
if (!initPTS[frag.cc] && maxAvCC === -1) {
unparsedVttFrags.push(data);
return;
}

const hls = this.hls;
// Parse the WebVTT file contents.
const payloadWebVTT = frag.initSegment?.data
Expand All @@ -562,7 +556,7 @@ export class TimelineController implements ComponentAPI {
parseWebVTT(
payloadWebVTT,
this.initPTS[frag.cc],
vttCCs,
this.vttCCs,
frag.cc,
frag.start,
(cues) => {
Expand All @@ -573,9 +567,18 @@ export class TimelineController implements ComponentAPI {
});
},
(error) => {
this._fallbackToIMSC1(frag, payload);
const missingInitPTS =
error.message === 'Missing initPTS for VTT MPEGTS';
if (missingInitPTS) {
unparsedVttFrags.push(data);
} else {
this._fallbackToIMSC1(frag, payload);
}
// Something went wrong while parsing. Trigger event with success false.
logger.log(`Failed to parse VTT cue: ${error}`);
if (missingInitPTS && maxAvCC > frag.cc) {
return;
}
hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, {
success: false,
frag: frag,
Expand Down Expand Up @@ -631,10 +634,6 @@ export class TimelineController implements ComponentAPI {
) {
const { frag } = data;
if (frag.type === PlaylistLevelType.SUBTITLE) {
if (!this.initPTS[frag.cc]) {
this.unparsedVttFrags.push(data as unknown as FragLoadedData);
return;
}
this.onFragLoaded(Events.FRAG_LOADED, data as unknown as FragLoadedData);
}
}
Expand Down
14 changes: 8 additions & 6 deletions src/utils/webvtt-parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ const calculateOffset = function (vttCCs: VTTCCs, cc, presentationTime) {

export function parseWebVTT(
vttByteArray: ArrayBuffer,
initPTS: RationalTimestamp,
initPTS: RationalTimestamp | undefined,
vttCCs: VTTCCs,
cc: number,
timeOffset: number,
Expand All @@ -107,10 +107,9 @@ export function parseWebVTT(
.replace(LINEBREAKS, '\n')
.split('\n');
const cues: VTTCue[] = [];
const init90kHz = toMpegTsClockFromTimescale(
initPTS.baseTime,
initPTS.timescale
);
const init90kHz = initPTS
? toMpegTsClockFromTimescale(initPTS.baseTime, initPTS.timescale)
: 0;
let cueTime = '00:00.000';
let timestampMapMPEGTS = 0;
let timestampMapLOCAL = 0;
Expand All @@ -134,8 +133,11 @@ export function parseWebVTT(
calculateOffset(vttCCs, cc, webVttMpegTsMapOffset);
}
}

if (webVttMpegTsMapOffset) {
if (!initPTS) {
parsingError = new Error('Missing initPTS for VTT MPEGTS');
return;
}
// If we have MPEGTS, offset = presentation time + discontinuity offset
cueOffset = webVttMpegTsMapOffset - vttCCs.presentationOffset;
}
Expand Down

0 comments on commit 7002632

Please sign in to comment.