Skip to content

Commit

Permalink
Align live WebVTT subtitle playlists with main variant using media-se…
Browse files Browse the repository at this point in the history 
…quence when program-date-time is unavailable

Resolves #3987
  • Loading branch information
Rob Walch committed Jun 3, 2021
1 parent 4a29630 commit 5e8468a
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 29 deletions.
12 changes: 6 additions & 6 deletions src/controller/audio-stream-controller.ts
View file
@@ -1,20 +1,20 @@
import BaseStreamController, { State } from './base-stream-controller';
import type { NetworkComponentAPI } from '../types/component-api';
import { Events } from '../events';
import { BufferHelper } from '../utils/buffer-helper';
import type { FragmentTracker } from './fragment-tracker';
import { FragmentState } from './fragment-tracker';
import { Level } from '../types/level';
import { PlaylistLevelType } from '../types/loader';
import { Fragment, ElementaryStreamTypes, Part } from '../loader/fragment';
import ChunkCache from '../demux/chunk-cache';
import TransmuxerInterface from '../demux/transmuxer-interface';
import type { TransmuxerResult } from '../types/transmuxer';
import { ChunkMetadata } from '../types/transmuxer';
import { fragmentWithinToleranceTest } from './fragment-finders';
import { alignPDT } from '../utils/discontinuities';
import { ErrorDetails } from '../errors';
import { logger } from '../utils/logger';
import type { NetworkComponentAPI } from '../types/component-api';
import type { FragmentTracker } from './fragment-tracker';
import type { TransmuxerResult } from '../types/transmuxer';
import type Hls from '../hls';
import type { LevelDetails } from '../loader/level-details';
import type { TrackSet } from '../types/track';
Expand All @@ -31,8 +31,8 @@ import type {
FragParsingMetadataData,
FragParsingUserdataData,
FragBufferedData,
ErrorData,
} from '../types/events';
import type { ErrorData } from '../types/events';

const TICK_INTERVAL = 100; // how often to tick in ms

Expand Down Expand Up @@ -453,8 +453,8 @@ class AudioStreamController
}
if (
!track.details &&
this.mainDetails?.hasProgramDateTime &&
newDetails.hasProgramDateTime
newDetails.hasProgramDateTime &&
this.mainDetails?.hasProgramDateTime
) {
alignPDT(newDetails, this.mainDetails);
sliding = newDetails.fragments[0].start;
Expand Down
80 changes: 58 additions & 22 deletions src/controller/subtitle-stream-controller.ts
View file
Expand Up @@ -2,11 +2,13 @@ import { Events } from '../events';
import { logger } from '../utils/logger';
import { BufferHelper } from '../utils/buffer-helper';
import { findFragmentByPDT, findFragmentByPTS } from './fragment-finders';
import type { FragmentTracker } from './fragment-tracker';
import { alignPDT } from '../utils/discontinuities';
import { adjustSliding } from './level-helper';
import { FragmentState } from './fragment-tracker';
import BaseStreamController, { State } from './base-stream-controller';
import { PlaylistLevelType } from '../types/loader';
import { Level } from '../types/level';
import type { FragmentTracker } from './fragment-tracker';
import type { NetworkComponentAPI } from '../types/component-api';
import type Hls from '../hls';
import type { LevelDetails } from '../loader/level-details';
Expand All @@ -19,6 +21,7 @@ import type {
TrackLoadedData,
TrackSwitchedData,
BufferFlushingData,
LevelLoadedData,
} from '../types/events';

const TICK_INTERVAL = 500; // how often to tick in ms
Expand All @@ -36,16 +39,24 @@ export class SubtitleStreamController

private currentTrackId: number = -1;
private tracksBuffered: Array<TimeRange[]> = [];
private mainDetails: LevelDetails | null = null;

constructor(hls: Hls, fragmentTracker: FragmentTracker) {
super(hls, fragmentTracker, '[subtitle-stream-controller]');
this._registerListeners();
}

protected onHandlerDestroying() {
this._unregisterListeners();
this.mainDetails = null;
}

private _registerListeners() {
const { hls } = this;
hls.on(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
hls.on(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
hls.on(Events.MANIFEST_LOADING, this.onManifestLoading, this);
hls.on(Events.LEVEL_LOADED, this.onLevelLoaded, this);
hls.on(Events.ERROR, this.onError, this);
hls.on(Events.SUBTITLE_TRACKS_UPDATED, this.onSubtitleTracksUpdated, this);
hls.on(Events.SUBTITLE_TRACK_SWITCH, this.onSubtitleTrackSwitch, this);
Expand All @@ -58,6 +69,8 @@ export class SubtitleStreamController
const { hls } = this;
hls.off(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
hls.off(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
hls.off(Events.MANIFEST_LOADING, this.onManifestLoading, this);
hls.off(Events.LEVEL_LOADED, this.onLevelLoaded, this);
hls.off(Events.ERROR, this.onError, this);
hls.off(Events.SUBTITLE_TRACKS_UPDATED, this.onSubtitleTracksUpdated, this);
hls.off(Events.SUBTITLE_TRACK_SWITCH, this.onSubtitleTrackSwitch, this);
Expand All @@ -70,17 +83,28 @@ export class SubtitleStreamController
this.stopLoad();
this.state = State.IDLE;

// Check if we already have a track with necessary details to load fragments
const currentTrack = this.levels[this.currentTrackId];
if (currentTrack?.details) {
this.setInterval(TICK_INTERVAL);
this.tick();
}
this.setInterval(TICK_INTERVAL);
this.tick();
}

onHandlerDestroyed() {
this._unregisterListeners();
super.onHandlerDestroyed();
onManifestLoading() {
this.mainDetails = null;
this.fragmentTracker.removeAllFragments();
}

onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) {
const mainDetails = data.details;
if (this.mainDetails === null) {
const trackId = this.levelLastLoaded;
if (trackId !== null && this.levels && mainDetails.live) {
const track = this.levels[trackId];
if (!track.details || !track.details.fragments[0]) {
return;
}
alignPDT(track.details, mainDetails);
}
}
this.mainDetails = mainDetails;
}

onSubtitleFragProcessed(
Expand Down Expand Up @@ -207,27 +231,39 @@ export class SubtitleStreamController
event: Events.SUBTITLE_TRACK_LOADED,
data: TrackLoadedData
) {
const { id, details } = data;
const { details: newDetails, id: trackId } = data;
const { currentTrackId, levels } = this;
if (!levels.length || !details) {
if (!levels.length) {
return;
}
const currentTrack: Level = levels[currentTrackId];
if (id >= levels.length || id !== currentTrackId || !currentTrack) {
const track: Level = levels[currentTrackId];
if (trackId >= levels.length || trackId !== currentTrackId || !track) {
return;
}
this.mediaBuffer = this.mediaBufferTimeRanges;
if (details.live || currentTrack.details?.live) {
if (details.deltaUpdateFailed) {
if (newDetails.live || track.details?.live) {
if (newDetails.deltaUpdateFailed) {
return;
}
// TODO: Subtitle Fragments should be assigned startPTS and endPTS once VTT/TTML is parsed
// otherwise this depends on DISCONTINUITY or PROGRAM-DATE-TIME tags to align playlists
this.alignPlaylists(details, currentTrack.details);
if (!track.details) {
const mainDetails = this.mainDetails;
if (mainDetails) {
if (newDetails.hasProgramDateTime && mainDetails.hasProgramDateTime) {
alignPDT(newDetails, mainDetails);
} else {
// try aligning on SN
adjustSliding(mainDetails, newDetails);
}
}
} else {
this.alignPlaylists(newDetails, track.details);
}
}
currentTrack.details = details;
this.levelLastLoaded = id;
this.setInterval(TICK_INTERVAL);
track.details = newDetails;
this.levelLastLoaded = trackId;

// trigger handler right now
this.tick();
}

_handleFragmentLoadComplete(fragLoadedData: FragLoadedData) {
Expand Down
1 change: 0 additions & 1 deletion tests/unit/controller/subtitle-stream-controller.js
View file
Expand Up @@ -98,7 +98,6 @@ describe('SubtitleStreamController', function () {
details: details,
});
expect(subtitleStreamController.levels[1].details).to.equal(details);
expect(subtitleStreamController.setInterval).to.have.been.calledOnce;
});

it('should ignore the event if the data does not match the current track', function () {
Expand Down

0 comments on commit 5e8468a

Please sign in to comment.