From a09a61f4cdb294285a73f501406604007a28e574 Mon Sep 17 00:00:00 2001 From: Peter Tseng Date: Mon, 19 Apr 2021 19:34:27 -0700 Subject: [PATCH] #3312 support multiple EXT-X-MAP tags; temporarily disable sidx loading --- api-extractor/report/hls.js.api.md | 6 +- src/controller/audio-stream-controller.ts | 105 ++++++++++----------- src/controller/base-stream-controller.ts | 33 ++++--- src/controller/level-helper.ts | 12 +-- src/controller/stream-controller.ts | 109 +++++++++++----------- src/loader/fragment.ts | 2 + src/loader/level-details.ts | 4 +- src/loader/m3u8-parser.ts | 51 +++++----- src/loader/playlist-loader.ts | 41 ++++---- tests/unit/loader/playlist-loader.js | 71 ++++++++++---- 10 files changed, 241 insertions(+), 193 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 2ef051373b8..6d7eb4b00ff 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -715,6 +715,8 @@ export class Fragment extends BaseSegment { // (undocumented) endPTS?: number; // (undocumented) + initSegment: Fragment | null; + // (undocumented) level: number; // (undocumented) levelkey?: LevelKey; @@ -1344,8 +1346,6 @@ export class LevelDetails { // (undocumented) holdBack: number; // (undocumented) - initSegment: Fragment | null; - // (undocumented) get lastPartIndex(): number; // (undocumented) get lastPartSn(): number; @@ -1358,8 +1358,6 @@ export class LevelDetails { // (undocumented) misses: number; // (undocumented) - needSidxRanges: boolean; - // (undocumented) get partEnd(): number; // (undocumented) partHoldBack: number; diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index c63423e3c3b..e3846792de6 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -281,67 +281,63 @@ class AudioStreamController return; } - let frag = trackDetails.initSegment; + // let frag = trackDetails.initSegment; let targetBufferTime = 0; - if (!frag || frag.data) { - const mediaBuffer = this.mediaBuffer ? this.mediaBuffer : this.media; - const videoBuffer = this.videoBuffer ? this.videoBuffer : this.media; - const maxBufferHole = - pos < config.maxBufferHole - ? Math.max(MAX_START_GAP_JUMP, config.maxBufferHole) - : config.maxBufferHole; - const bufferInfo = BufferHelper.bufferInfo( - mediaBuffer, - pos, - maxBufferHole - ); - const mainBufferInfo = BufferHelper.bufferInfo( - videoBuffer, - pos, - maxBufferHole - ); - const bufferLen = bufferInfo.len; - const maxConfigBuffer = Math.min( - config.maxBufferLength, - config.maxMaxBufferLength - ); - const maxBufLen = Math.max(maxConfigBuffer, mainBufferInfo.len); - const audioSwitch = this.audioSwitch; + // if (!frag || frag.data) { + const mediaBuffer = this.mediaBuffer ? this.mediaBuffer : this.media; + const videoBuffer = this.videoBuffer ? this.videoBuffer : this.media; + const maxBufferHole = + pos < config.maxBufferHole + ? Math.max(MAX_START_GAP_JUMP, config.maxBufferHole) + : config.maxBufferHole; + const bufferInfo = BufferHelper.bufferInfo(mediaBuffer, pos, maxBufferHole); + const mainBufferInfo = BufferHelper.bufferInfo( + videoBuffer, + pos, + maxBufferHole + ); + const bufferLen = bufferInfo.len; + const maxConfigBuffer = Math.min( + config.maxBufferLength, + config.maxMaxBufferLength + ); + const maxBufLen = Math.max(maxConfigBuffer, mainBufferInfo.len); + const audioSwitch = this.audioSwitch; - // if buffer length is less than maxBufLen try to load a new fragment - if (bufferLen >= maxBufLen && !audioSwitch) { - return; - } + // if buffer length is less than maxBufLen try to load a new fragment + if (bufferLen >= maxBufLen && !audioSwitch) { + return; + } - if (!audioSwitch && this._streamEnded(bufferInfo, trackDetails)) { - hls.trigger(Events.BUFFER_EOS, { type: 'audio' }); - this.state = State.ENDED; - return; - } + if (!audioSwitch && this._streamEnded(bufferInfo, trackDetails)) { + hls.trigger(Events.BUFFER_EOS, { type: 'audio' }); + this.state = State.ENDED; + return; + } - const fragments = trackDetails.fragments; - const start = fragments[0].start; - targetBufferTime = bufferInfo.end; - - if (audioSwitch) { - targetBufferTime = pos; - // if currentTime (pos) is less than alt audio playlist start time, it means that alt audio is ahead of currentTime - if (trackDetails.PTSKnown && pos < start) { - // if everything is buffered from pos to start or if audio buffer upfront, let's seek to start - if (bufferInfo.end > start || bufferInfo.nextStart) { - this.log( - 'Alt audio track ahead of main track, seek to start of alt audio track' - ); - media.currentTime = start + 0.05; - } + const fragments = trackDetails.fragments; + const start = fragments[0].start; + targetBufferTime = bufferInfo.end; + + if (audioSwitch) { + targetBufferTime = pos; + // if currentTime (pos) is less than alt audio playlist start time, it means that alt audio is ahead of currentTime + if (trackDetails.PTSKnown && pos < start) { + // if everything is buffered from pos to start or if audio buffer upfront, let's seek to start + if (bufferInfo.end > start || bufferInfo.nextStart) { + this.log( + 'Alt audio track ahead of main track, seek to start of alt audio track' + ); + media.currentTime = start + 0.05; } } + } - frag = this.getNextFragment(targetBufferTime, trackDetails); - if (!frag) { - return; - } + const frag = this.getNextFragment(targetBufferTime, trackDetails); + if (!frag) { + return; } + // } if (frag.decryptdata?.keyFormat === 'identity' && !frag.decryptdata?.key) { this.loadKey(frag, trackDetails); @@ -506,7 +502,8 @@ class AudioStreamController // Check if we have video initPTS // If not we need to wait for it const initPTS = this.initPTS[frag.cc]; - const initSegmentData = details.initSegment?.data; + // const initSegmentData = details.initSegment?.data; + const initSegmentData = frag.initSegment?.data; if (initPTS !== undefined) { // this.log(`Transmuxing ${sn} of [${details.startSN} ,${details.endSN}],track ${trackId}`); // time Offset is accurate if level PTS is known, or if playlist is not sliding (not live) diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 1fdc62e73a6..999c26ac9f4 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -424,16 +424,17 @@ export default class BaseStreamController details, 'Level details are defined when init segment is loaded' ); - const initSegment = details.initSegment as Fragment; - console.assert( - initSegment, - 'Fragment initSegment is defined when init segment is loaded' - ); + // const initSegment = details.initSegment as Fragment; + // console.assert( + // initSegment, + // 'Fragment initSegment is defined when init segment is loaded' + // ); const stats = frag.stats; this.state = State.IDLE; this.fragLoadError = 0; - initSegment.data = new Uint8Array(data.payload); + // initSegment.data = new Uint8Array(data.payload); + frag.data = new Uint8Array(data.payload); stats.parsing.start = stats.buffering.start = self.performance.now(); stats.parsing.end = stats.buffering.end = self.performance.now(); @@ -752,13 +753,14 @@ export default class BaseStreamController let frag; // If an initSegment is present, it must be buffered first - if ( - levelDetails.initSegment && - !levelDetails.initSegment.data && - !this.bitrateTest - ) { - frag = levelDetails.initSegment; - } else if (levelDetails.live) { + // if ( + // levelDetails.initSegment && + // !levelDetails.initSegment.data && + // !this.bitrateTest + // ) { + // frag = levelDetails.initSegment; + // } else if (levelDetails.live) { + if (levelDetails.live) { const initialLiveManifestSize = config.initialLiveManifestSize; if (fragLen < initialLiveManifestSize) { this.warn( @@ -793,6 +795,11 @@ export default class BaseStreamController frag = this.getFragmentAtPosition(pos, end, levelDetails); } + // If an initSegment is present, it must be buffered first + if (frag.initSegment && !frag.initSegment.data && !this.bitrateTest) { + frag = frag.initSegment; + } + return frag; } diff --git a/src/controller/level-helper.ts b/src/controller/level-helper.ts index 49386fe1547..cb8e195340a 100644 --- a/src/controller/level-helper.ts +++ b/src/controller/level-helper.ts @@ -166,9 +166,9 @@ export function mergeDetails( newDetails: LevelDetails ): void { // potentially retrieve cached initsegment - if (newDetails.initSegment && oldDetails.initSegment) { - newDetails.initSegment = oldDetails.initSegment; - } + // if (newDetails.initSegment && oldDetails.initSegment) { + // newDetails.initSegment = oldDetails.initSegment; + // } if (oldDetails.fragmentHint) { // prevent PTS and duration from being adjusted on the next hint @@ -239,9 +239,9 @@ export function mergeDetails( } } if (newDetails.skippedSegments) { - if (!newDetails.initSegment) { - newDetails.initSegment = oldDetails.initSegment; - } + // if (!newDetails.initSegment) { + // newDetails.initSegment = oldDetails.initSegment; + // } newDetails.startCC = newDetails.fragments[0].cc; } diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 2cf0b6db48c..2974bac700f 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -244,63 +244,67 @@ export default class StreamController return; } - let frag = levelDetails.initSegment; + // let frag = levelDetails.initSegment; let targetBufferTime = 0; - if (!frag || frag.data || this.bitrateTest) { - // compute max Buffer Length that we could get from this load level, based on level bitrate. don't buffer more than 60 MB and more than 30s - const levelBitrate = levelInfo.maxBitrate; - let maxBufLen; - if (levelBitrate) { - maxBufLen = Math.max( - (8 * config.maxBufferSize) / levelBitrate, - config.maxBufferLength - ); - } else { - maxBufLen = config.maxBufferLength; - } - maxBufLen = Math.min(maxBufLen, config.maxMaxBufferLength); - - // determine next candidate fragment to be loaded, based on current position and end of buffer position - // ensure up to `config.maxMaxBufferLength` of buffer upfront - const maxBufferHole = - pos < config.maxBufferHole - ? Math.max(MAX_START_GAP_JUMP, config.maxBufferHole) - : config.maxBufferHole; - const bufferInfo = BufferHelper.bufferInfo( - this.mediaBuffer ? this.mediaBuffer : media, - pos, - maxBufferHole + // if (!frag || frag.data || this.bitrateTest) { + // compute max Buffer Length that we could get from this load level, based on level bitrate. don't buffer more than 60 MB and more than 30s + const levelBitrate = levelInfo.maxBitrate; + let maxBufLen; + if (levelBitrate) { + maxBufLen = Math.max( + (8 * config.maxBufferSize) / levelBitrate, + config.maxBufferLength ); - const bufferLen = bufferInfo.len; - // Stay idle if we are still with buffer margins - if (bufferLen >= maxBufLen) { - return; + } else { + maxBufLen = config.maxBufferLength; + } + maxBufLen = Math.min(maxBufLen, config.maxMaxBufferLength); + + // determine next candidate fragment to be loaded, based on current position and end of buffer position + // ensure up to `config.maxMaxBufferLength` of buffer upfront + const maxBufferHole = + pos < config.maxBufferHole + ? Math.max(MAX_START_GAP_JUMP, config.maxBufferHole) + : config.maxBufferHole; + const bufferInfo = BufferHelper.bufferInfo( + this.mediaBuffer ? this.mediaBuffer : media, + pos, + maxBufferHole + ); + const bufferLen = bufferInfo.len; + // Stay idle if we are still with buffer margins + if (bufferLen >= maxBufLen) { + return; + } + + if (this._streamEnded(bufferInfo, levelDetails)) { + const data: BufferEOSData = {}; + if (this.altAudio) { + data.type = 'video'; } - if (this._streamEnded(bufferInfo, levelDetails)) { - const data: BufferEOSData = {}; - if (this.altAudio) { - data.type = 'video'; - } + this.hls.trigger(Events.BUFFER_EOS, data); + this.state = State.ENDED; + return; + } - this.hls.trigger(Events.BUFFER_EOS, data); - this.state = State.ENDED; - return; - } + targetBufferTime = bufferInfo.end; + let frag = this.getNextFragment(targetBufferTime, levelDetails); + // Avoid loop loading by using nextLoadPosition set for backtracking + if ( + frag && + this.fragmentTracker.getState(frag) === FragmentState.OK && + this.nextLoadPosition > targetBufferTime + ) { + frag = this.getNextFragment(this.nextLoadPosition, levelDetails); + } + if (!frag) { + return; + } + // } - targetBufferTime = bufferInfo.end; - frag = this.getNextFragment(targetBufferTime, levelDetails); - // Avoid loop loading by using nextLoadPosition set for backtracking - if ( - frag && - this.fragmentTracker.getState(frag) === FragmentState.OK && - this.nextLoadPosition > targetBufferTime - ) { - frag = this.getNextFragment(this.nextLoadPosition, levelDetails); - } - if (!frag) { - return; - } + if (frag.initSegment && !frag.initSegment.data && !this.bitrateTest) { + frag = frag.initSegment; } // We want to load the key if we're dealing with an identity key, because we will decrypt @@ -671,7 +675,8 @@ export default class StreamController // time Offset is accurate if level PTS is known, or if playlist is not sliding (not live) const accurateTimeOffset = details.PTSKnown || !details.live; - const initSegmentData = details.initSegment?.data; + // const initSegmentData = details.initSegment?.data; + const initSegmentData = frag.initSegment?.data; const audioCodec = this._getAudioCodec(currentLevel); // transmux the MPEG-TS data to ISO-BMFF segments diff --git a/src/loader/fragment.ts b/src/loader/fragment.ts index 235555dfb85..2782b07240f 100644 --- a/src/loader/fragment.ts +++ b/src/loader/fragment.ts @@ -139,6 +139,8 @@ export class Fragment extends BaseSegment { public bitrateTest: boolean = false; // #EXTINF segment title public title: string | null = null; + // The Media Initialization Section for this segment + public initSegment: Fragment | null = null; constructor(type: PlaylistLevelType, baseurl: string) { super(baseurl); diff --git a/src/loader/level-details.ts b/src/loader/level-details.ts index faee20f7363..c9e245672c9 100644 --- a/src/loader/level-details.ts +++ b/src/loader/level-details.ts @@ -13,7 +13,7 @@ export class LevelDetails { public fragments: Fragment[]; public fragmentHint?: Fragment; public partList: Part[] | null = null; - public initSegment: Fragment | null = null; + // public initSegment: Fragment | null = null; public live: boolean = true; public ageHeader: number = 0; public advancedDateTime?: number; @@ -21,7 +21,7 @@ export class LevelDetails { public advanced: boolean = true; public availabilityDelay?: number; // Manifest reload synchronization public misses: number = 0; - public needSidxRanges: boolean = false; + // public needSidxRanges: boolean = false; public startCC: number = 0; public startSN: number = 0; public startTimeOffset: number | null = null; diff --git a/src/loader/m3u8-parser.ts b/src/loader/m3u8-parser.ts index 8300d799093..c5cb6812266 100644 --- a/src/loader/m3u8-parser.ts +++ b/src/loader/m3u8-parser.ts @@ -198,6 +198,8 @@ export default class M3U8Parser { ): LevelDetails { const level = new LevelDetails(baseurl); const fragments: M3U8ParserFragments = level.fragments; + // The most recent init segment seen (applies to all subsequent segments) + let currentInitSegment: Fragment | null = null; let currentSN = 0; let currentPart = 0; let totalduration = 0; @@ -232,6 +234,9 @@ export default class M3U8Parser { frag.level = id; frag.cc = discontinuityCounter; frag.urlId = levelUrlId; + if (currentInitSegment) { + frag.initSegment = currentInitSegment; + } fragments.push(frag); // avoid sliced strings https://github.com/video-dev/hls.js/issues/939 frag.relurl = (' ' + result[3]).slice(1); @@ -426,9 +431,11 @@ export default class M3U8Parser { if (levelkey) { frag.levelkey = levelkey; } - level.initSegment = frag; + // level.initSegment = frag; + currentInitSegment = frag; frag = new Fragment(type, baseurl); - frag.rawProgramDateTime = level.initSegment.rawProgramDateTime; + // frag.rawProgramDateTime = level.initSegment.rawProgramDateTime; + frag.rawProgramDateTime = currentInitSegment.rawProgramDateTime; break; } case 'SERVER-CONTROL': { @@ -510,26 +517,26 @@ export default class M3U8Parser { level.endSN = lastSn !== 'initSegment' ? lastSn : 0; if (firstFragment) { level.startCC = firstFragment.cc; - if (!level.initSegment) { - // this is a bit lurky but HLS really has no other way to tell us - // if the fragments are TS or MP4, except if we download them :/ - // but this is to be able to handle SIDX. - if ( - level.fragments.every( - (frag) => frag.relurl && isMP4Url(frag.relurl) - ) - ) { - logger.warn( - 'MP4 fragments found but no init segment (probably no MAP, incomplete M3U8), trying to fetch SIDX' - ); - frag = new Fragment(type, baseurl); - frag.relurl = lastFragment.relurl; - frag.level = id; - frag.sn = 'initSegment'; - level.initSegment = frag; - level.needSidxRanges = true; - } - } + // if (!level.initSegment) { + // // this is a bit lurky but HLS really has no other way to tell us + // // if the fragments are TS or MP4, except if we download them :/ + // // but this is to be able to handle SIDX. + // if ( + // level.fragments.every( + // (frag) => frag.relurl && isMP4Url(frag.relurl) + // ) + // ) { + // logger.warn( + // 'MP4 fragments found but no init segment (probably no MAP, incomplete M3U8), trying to fetch SIDX' + // ); + // frag = new Fragment(type, baseurl); + // frag.relurl = lastFragment.relurl; + // frag.level = id; + // frag.sn = 'initSegment'; + // level.initSegment = frag; + // level.needSidxRanges = true; + // } + // } } } else { level.endSN = 0; diff --git a/src/loader/playlist-loader.ts b/src/loader/playlist-loader.ts index 0eb16f62a3a..5af4a04405e 100644 --- a/src/loader/playlist-loader.ts +++ b/src/loader/playlist-loader.ts @@ -516,23 +516,23 @@ class PlaylistLoader { // in case we need SIDX ranges // return early after calling load for // the SIDX box. - if (levelDetails.needSidxRanges) { - const sidxUrl = (levelDetails.initSegment as Fragment).url as string; - this.load({ - url: sidxUrl, - isSidxRequest: true, - type, - level, - levelDetails, - id, - groupId: null, - rangeStart: 0, - rangeEnd: 2048, - responseType: 'arraybuffer', - deliveryDirectives: null, - }); - return; - } + // if (levelDetails.needSidxRanges) { + // const sidxUrl = (levelDetails.initSegment as Fragment).url as string; + // this.load({ + // url: sidxUrl, + // isSidxRequest: true, + // type, + // level, + // levelDetails, + // id, + // groupId: null, + // rangeStart: 0, + // rangeEnd: 2048, + // responseType: 'arraybuffer', + // deliveryDirectives: null, + // }); + // return; + // } // extend the context with the new levelDetails property context.levelDetails = levelDetails; @@ -565,9 +565,10 @@ class PlaylistLoader { ); } }); - (levelDetails.initSegment as Fragment).setByteRange( - String(sidxInfo.moovEndOffset) + '@0' - ); + console.assert(false, 'Unexpected sidx request'); + // (levelDetails.initSegment as Fragment).setByteRange( + // String(sidxInfo.moovEndOffset) + '@0' + // ); } private handleManifestParsingError( diff --git a/tests/unit/loader/playlist-loader.js b/tests/unit/loader/playlist-loader.js index c1b06603035..1ac24dccefb 100644 --- a/tests/unit/loader/playlist-loader.js +++ b/tests/unit/loader/playlist-loader.js @@ -344,22 +344,22 @@ http://proxy-62.dailymotion.com/sec(3ae40f708f79ca9471f52b86da76a3a8)/video/107/ ); }); - it('handles a missing init segment for mp4 segment urls', function () { - const level = `#EXTM3U -#EXT-X-VERSION:3 -#EXT-X-PLAYLIST-TYPE:VOD -#EXT-X-TARGETDURATION:14 -#EXTINF:11.360, -/something.mp4?abc -#EXT-X-ENDLIST`; - const result = M3U8Parser.parseLevelPlaylist( - level, - 'http://example.invalid/playlist.m3u8', - 0 - ); - expect(result.initSegment).to.be.ok; - expect(result.initSegment.relurl).to.equal('/something.mp4?abc'); - }); + // it('handles a missing init segment for mp4 segment urls', function () { + // const level = `#EXTM3U + // #EXT-X-VERSION:3 + // #EXT-X-PLAYLIST-TYPE:VOD + // #EXT-X-TARGETDURATION:14 + // #EXTINF:11.360, + // /something.mp4?abc + // #EXT-X-ENDLIST`; + // const result = M3U8Parser.parseLevelPlaylist( + // level, + // 'http://example.invalid/playlist.m3u8', + // 0 + // ); + // expect(result.initSegment).to.be.ok; + // expect(result.initSegment.relurl).to.equal('/something.mp4?abc'); + // }); it('parse level with single char fragment URI', function () { const level = `#EXTM3U @@ -950,12 +950,43 @@ main.mp4`; 'http://proxy-62.dailymotion.com/sec(3ae40f708f79ca9471f52b86da76a3a8)/video/107/282/158282701_mp4_h264_aac_hq.m3u8#cell=core', 0 ); - expect(result.initSegment.url).to.equal( + const initSegment = result.fragments[0].initSegment; + expect(initSegment.url).to.equal( 'http://proxy-62.dailymotion.com/sec(3ae40f708f79ca9471f52b86da76a3a8)/video/107/282/main.mp4' ); - expect(result.initSegment.byteRangeStartOffset).to.equal(0); - expect(result.initSegment.byteRangeEndOffset).to.equal(718); - expect(result.initSegment.sn).to.equal('initSegment'); + expect(initSegment.byteRangeStartOffset).to.equal(0); + expect(initSegment.byteRangeEndOffset).to.equal(718); + expect(initSegment.sn).to.equal('initSegment'); + }); + + it('parses multiple #EXT-X-MAP URI', function () { + const level = `#EXTM3U +#EXT-X-TARGETDURATION:6 +#EXT-X-VERSION:7 +#EXT-X-MEDIA-SEQUENCE:1 +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-INDEPENDENT-SEGMENTS +#EXT-X-MAP:URI="main.mp4" +#EXTINF:6.00600, +frag1.mp4 +#EXT-X-DISCONTINUITY +#EXT-X-MAP:URI="alt.mp4" +#EXTINF:4.0 +frag2.mp4 +`; + const result = M3U8Parser.parseLevelPlaylist( + level, + 'http://video.example.com/disc.m3u8', + 0 + ); + expect(result.fragments[0].initSegment.url).to.equal( + 'http://video.example.com/main.mp4' + ); + expect(result.fragments[0].initSegment.sn).to.equal('initSegment'); + expect(result.fragments[1].initSegment.url).to.equal( + 'http://video.example.com/alt.mp4' + ); + expect(result.fragments[1].initSegment.sn).to.equal('initSegment'); }); describe('PDT calculations', function () {