From 1a8dd7108bca050a8692473be50bc4bfeb4a8489 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Thu, 27 May 2021 17:00:37 -0400 Subject: [PATCH] Fix forward buffer length estimation when in a gap, and fix buffer controller after append error Resolves #3914 --- src/controller/audio-stream-controller.ts | 81 ++++++++++-------- src/controller/base-stream-controller.ts | 49 ++++++++++- src/controller/buffer-operation-queue.ts | 1 + src/controller/stream-controller.ts | 89 +++++++++----------- src/controller/subtitle-stream-controller.ts | 12 ++- src/utils/buffer-helper.ts | 2 +- 6 files changed, 139 insertions(+), 95 deletions(-) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 5d4b92700ed..684aff8a89f 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -13,7 +13,6 @@ import type { TransmuxerResult } from '../types/transmuxer'; import { ChunkMetadata } from '../types/transmuxer'; import { fragmentWithinToleranceTest } from './fragment-finders'; import { alignPDT } from '../utils/discontinuities'; -import { MAX_START_GAP_JUMP } from './gap-controller'; import { ErrorDetails } from '../errors'; import { logger } from '../utils/logger'; import type Hls from '../hls'; @@ -55,6 +54,7 @@ class AudioStreamController private trackId: number = -1; private waitingData: WaitingForPTSData | null = null; private mainDetails: LevelDetails | null = null; + private bufferFlushed: boolean = false; constructor(hls: Hls, fragmentTracker: FragmentTracker) { super(hls, fragmentTracker, '[audio-stream-controller]'); @@ -270,11 +270,6 @@ class AudioStreamController return; } - const pos = this.getLoadPosition(); - if (!Number.isFinite(pos)) { - return; - } - const levelInfo = levels[trackId]; const trackDetails = levelInfo.details; @@ -287,25 +282,24 @@ class AudioStreamController return; } - let targetBufferTime = 0; - const mediaBuffer = this.mediaBuffer ? this.mediaBuffer : this.media; - const videoBuffer = this.videoBuffer ? this.videoBuffer : this.media; - const maxBufferHole = - pos < config.maxBufferHole - ? Math.max(MAX_START_GAP_JUMP, config.maxBufferHole) - : config.maxBufferHole; - const bufferInfo = BufferHelper.bufferInfo(mediaBuffer, pos, maxBufferHole); - const mainBufferInfo = BufferHelper.bufferInfo( - videoBuffer, - pos, - maxBufferHole + if (this.bufferFlushed) { + this.bufferFlushed = false; + this.afterBufferFlushed( + this.mediaBuffer ? this.mediaBuffer : this.media, + ElementaryStreamTypes.AUDIO, + PlaylistLevelType.AUDIO + ); + } + + const bufferInfo = this.getFwdBufferInfo( + this.mediaBuffer ? this.mediaBuffer : this.media, + PlaylistLevelType.AUDIO ); + if (bufferInfo === null) { + return; + } const bufferLen = bufferInfo.len; - const maxConfigBuffer = Math.min( - config.maxBufferLength, - config.maxMaxBufferLength - ); - const maxBufLen = Math.max(maxConfigBuffer, mainBufferInfo.len); + const maxBufLen = this.getMaxBufferLength(); const audioSwitch = this.audioSwitch; // if buffer length is less than maxBufLen try to load a new fragment @@ -321,9 +315,10 @@ class AudioStreamController const fragments = trackDetails.fragments; const start = fragments[0].start; - targetBufferTime = bufferInfo.end; + let targetBufferTime = bufferInfo.end; if (audioSwitch) { + const pos = this.getLoadPosition(); targetBufferTime = pos; // if currentTime (pos) is less than alt audio playlist start time, it means that alt audio is ahead of currentTime if (trackDetails.PTSKnown && pos < start) { @@ -339,6 +334,7 @@ class AudioStreamController const frag = this.getNextFragment(targetBufferTime, trackDetails); if (!frag) { + this.bufferFlushed = true; return; } @@ -349,6 +345,18 @@ class AudioStreamController } } + protected getMaxBufferLength(): number { + const maxConfigBuffer = super.getMaxBufferLength(); + const mainBufferInfo = this.getFwdBufferInfo( + this.videoBuffer ? this.videoBuffer : this.media, + PlaylistLevelType.MAIN + ); + if (mainBufferInfo === null) { + return maxConfigBuffer; + } + return Math.max(maxConfigBuffer, mainBufferInfo.len); + } + onMediaDetaching() { this.videoBuffer = null; super.onMediaDetaching(); @@ -399,6 +407,7 @@ class AudioStreamController this.mainDetails = null; this.fragmentTracker.removeAllFragments(); this.startPosition = this.lastCurrentTime = 0; + this.bufferFlushed = false; } onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) { @@ -625,17 +634,17 @@ class AudioStreamController data.parent === 'audio' && (this.state === State.PARSING || this.state === State.PARSED) ) { - const media = this.mediaBuffer; - const currentTime = this.media.currentTime; - const mediaBuffered = - media && - BufferHelper.isBuffered(media, currentTime) && - BufferHelper.isBuffered(media, currentTime + 0.5); + let flushBuffer = true; + const bufferedInfo = this.getFwdBufferInfo( + this.mediaBuffer, + PlaylistLevelType.AUDIO + ); + // 0.5 : tolerance needed as some browsers stalls playback before reaching buffered end // reduce max buf len if current position is buffered - if (mediaBuffered) { - this.reduceMaxBufferLength(); - this.state = State.IDLE; - } else { + if (bufferedInfo && bufferedInfo.len > 0.5) { + flushBuffer = !this.reduceMaxBufferLength(bufferedInfo.len); + } + if (flushBuffer) { // current position is not buffered, but browser is still complaining about buffer full error // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708 // in that case flush the whole audio buffer to recover @@ -645,6 +654,7 @@ class AudioStreamController this.fragCurrent = null; super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio'); } + this.resetLoadingState(); } break; default: @@ -657,8 +667,7 @@ class AudioStreamController { type }: BufferFlushedData ) { if (type === ElementaryStreamTypes.AUDIO) { - const media = this.mediaBuffer ? this.mediaBuffer : this.media; - this.afterBufferFlushed(media, type, PlaylistLevelType.AUDIO); + this.bufferFlushed = true; } } diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 2a166893019..c30a93d87f5 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -729,6 +729,53 @@ export default class BaseStreamController } } + protected getFwdBufferInfo( + bufferable: Bufferable, + type: PlaylistLevelType + ): { + len: number; + start: number; + end: number; + nextStart?: number; + } | null { + const { config } = this; + const pos = this.getLoadPosition(); + if (!Number.isFinite(pos)) { + return null; + } + const bufferInfo = BufferHelper.bufferInfo( + bufferable, + pos, + config.maxBufferHole + ); + // Workaround flaw in getting forward buffer when maxBufferHole is smaller than gap at current pos + if (bufferInfo.len === 0 && bufferInfo.nextStart !== undefined) { + const bufferedFragAtPos = this.fragmentTracker.getBufferedFrag(pos, type); + if (bufferedFragAtPos && bufferInfo.nextStart < bufferedFragAtPos.end) { + return BufferHelper.bufferInfo( + bufferable, + pos, + Math.max(bufferInfo.nextStart, config.maxBufferHole) + ); + } + } + return bufferInfo; + } + + protected getMaxBufferLength(levelBitrate?: number): number { + const { config } = this; + let maxBufLen; + if (levelBitrate) { + maxBufLen = Math.max( + (8 * config.maxBufferSize) / levelBitrate, + config.maxBufferLength + ); + } else { + maxBufLen = config.maxBufferLength; + } + return Math.min(maxBufLen, config.maxMaxBufferLength); + } + protected reduceMaxBufferLength(threshold?: number) { const config = this.config; const minLength = threshold || config.maxBufferLength; @@ -1105,7 +1152,7 @@ export default class BaseStreamController const { media } = this; // if we have not yet loaded any fragment, start loading from start position let pos = 0; - if (this.loadedmetadata) { + if (this.loadedmetadata && media) { pos = media.currentTime; } else if (this.nextLoadPosition) { pos = this.nextLoadPosition; diff --git a/src/controller/buffer-operation-queue.ts b/src/controller/buffer-operation-queue.ts index 6b8a0d2754d..250bccf31bc 100644 --- a/src/controller/buffer-operation-queue.ts +++ b/src/controller/buffer-operation-queue.ts @@ -67,6 +67,7 @@ export default class BufferOperationQueue { // Only shift the current operation off, otherwise the updateend handler will do this for us if (!sb || !sb.updating) { queue.shift(); + this.executeNext(type); } } } diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index d1d81358474..57d72c71e72 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -7,11 +7,11 @@ import type { FragmentTracker } from './fragment-tracker'; import { FragmentState } from './fragment-tracker'; import type { Level } from '../types/level'; import { PlaylistLevelType } from '../types/loader'; -import { Fragment, ElementaryStreamTypes } from '../loader/fragment'; +import { ElementaryStreamTypes, Fragment } from '../loader/fragment'; import TransmuxerInterface from '../demux/transmuxer-interface'; import type { TransmuxerResult } from '../types/transmuxer'; import { ChunkMetadata } from '../types/transmuxer'; -import GapController, { MAX_START_GAP_JUMP } from './gap-controller'; +import GapController from './gap-controller'; import { ErrorDetails } from '../errors'; import { logger } from '../utils/logger'; import type Hls from '../hls'; @@ -19,21 +19,21 @@ import type { LevelDetails } from '../loader/level-details'; import type { TrackSet } from '../types/track'; import type { SourceBufferName } from '../types/buffer'; import type { - MediaAttachedData, - BufferCreatedData, - ManifestParsedData, - LevelLoadingData, - LevelLoadedData, - LevelsUpdatedData, - AudioTrackSwitchingData, AudioTrackSwitchedData, + AudioTrackSwitchingData, + BufferCreatedData, + BufferEOSData, + BufferFlushedData, + ErrorData, + FragBufferedData, FragLoadedData, FragParsingMetadataData, FragParsingUserdataData, - FragBufferedData, - BufferFlushedData, - ErrorData, - BufferEOSData, + LevelLoadedData, + LevelLoadingData, + LevelsUpdatedData, + ManifestParsedData, + MediaAttachedData, } from '../types/events'; const TICK_INTERVAL = 100; // how often to tick in ms @@ -244,37 +244,18 @@ export default class StreamController return; } - const pos = this.getLoadPosition(); - if (!Number.isFinite(pos)) { + const bufferInfo = this.getFwdBufferInfo( + this.mediaBuffer ? this.mediaBuffer : media, + PlaylistLevelType.MAIN + ); + if (bufferInfo === null) { return; } + const bufferLen = bufferInfo.len; - let targetBufferTime = 0; // compute max Buffer Length that we could get from this load level, based on level bitrate. don't buffer more than 60 MB and more than 30s - const levelBitrate = levelInfo.maxBitrate; - let maxBufLen; - if (levelBitrate) { - maxBufLen = Math.max( - (8 * config.maxBufferSize) / levelBitrate, - config.maxBufferLength - ); - } else { - maxBufLen = config.maxBufferLength; - } - maxBufLen = Math.min(maxBufLen, config.maxMaxBufferLength); + const maxBufLen = this.getMaxBufferLength(levelInfo.maxBitrate); - // determine next candidate fragment to be loaded, based on current position and end of buffer position - // ensure up to `config.maxMaxBufferLength` of buffer upfront - const maxBufferHole = - pos < config.maxBufferHole - ? Math.max(MAX_START_GAP_JUMP, config.maxBufferHole) - : config.maxBufferHole; - const bufferInfo = BufferHelper.bufferInfo( - this.mediaBuffer ? this.mediaBuffer : media, - pos, - maxBufferHole - ); - const bufferLen = bufferInfo.len; // Stay idle if we are still with buffer margins if (bufferLen >= maxBufLen) { return; @@ -291,7 +272,7 @@ export default class StreamController return; } - targetBufferTime = bufferInfo.end; + const targetBufferTime = bufferInfo.end; let frag = this.getNextFragment(targetBufferTime, levelDetails); // Avoid backtracking after seeking or switching by loading an earlier segment in streams that could backtrack if ( @@ -312,6 +293,12 @@ export default class StreamController this.fragmentTracker.getState(frag) === FragmentState.OK && this.nextLoadPosition > targetBufferTime ) { + // Cleanup the fragment tracker before trying to find the next unbuffered fragment + const type = + this.audioOnly && !this.altAudio + ? ElementaryStreamTypes.AUDIO + : ElementaryStreamTypes.VIDEO; + this.afterBufferFlushed(media, type, PlaylistLevelType.MAIN); frag = this.getNextFragment(this.nextLoadPosition, levelDetails); } if (!frag) { @@ -885,25 +872,27 @@ export default class StreamController data.parent === 'main' && (this.state === State.PARSING || this.state === State.PARSED) ) { + let flushBuffer = true; + const bufferedInfo = this.getFwdBufferInfo( + this.media, + PlaylistLevelType.MAIN + ); // 0.5 : tolerance needed as some browsers stalls playback before reaching buffered end - const mediaBuffered = - !!this.media && - BufferHelper.isBuffered(this.media, this.media.currentTime) && - BufferHelper.isBuffered(this.media, this.media.currentTime + 0.5); // reduce max buf len if current position is buffered - if (mediaBuffered) { - this.reduceMaxBufferLength(); - this.state = State.IDLE; - } else { + if (bufferedInfo && bufferedInfo.len > 0.5) { + flushBuffer = !this.reduceMaxBufferLength(bufferedInfo.len); + } + if (flushBuffer) { // current position is not buffered, but browser is still complaining about buffer full error // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708 // in that case flush the whole buffer to recover this.warn( - 'buffer full error also media.currentTime is not buffered, flush everything' + 'buffer full error also media.currentTime is not buffered, flush main' ); - // flush everything + // flush main buffer this.immediateLevelSwitch(); } + this.resetLoadingState(); } break; default: diff --git a/src/controller/subtitle-stream-controller.ts b/src/controller/subtitle-stream-controller.ts index 7f76cb047c3..8bbbb65b63c 100644 --- a/src/controller/subtitle-stream-controller.ts +++ b/src/controller/subtitle-stream-controller.ts @@ -255,19 +255,16 @@ export class SubtitleStreamController return; } - const { maxBufferHole, maxFragLookUpTolerance } = config; - const maxConfigBuffer = Math.min( - config.maxBufferLength, - config.maxMaxBufferLength - ); const bufferedInfo = BufferHelper.bufferedInfo( this.mediaBufferTimeRanges, media.currentTime, - maxBufferHole + config.maxBufferHole ); const { end: targetBufferTime, len: bufferLen } = bufferedInfo; - if (bufferLen > maxConfigBuffer) { + const maxBufLen = this.getMaxBufferLength(); + + if (bufferLen > maxBufLen) { return; } @@ -284,6 +281,7 @@ export class SubtitleStreamController let foundFrag; const fragPrevious = this.fragPrevious; if (targetBufferTime < end) { + const { maxFragLookUpTolerance } = config; if (fragPrevious && trackDetails.hasProgramDateTime) { foundFrag = findFragmentByPDT( fragments, diff --git a/src/utils/buffer-helper.ts b/src/utils/buffer-helper.ts index 1a03489bf9c..f656c8eb66b 100644 --- a/src/utils/buffer-helper.ts +++ b/src/utils/buffer-helper.ts @@ -8,7 +8,7 @@ * Also @see https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/buffered */ -import { logger } from '../utils/logger'; +import { logger } from './logger'; type BufferTimeRange = { start: number;