From 0341fd7349af8f77d0dc2d387a8453deefcffd87 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Mon, 22 Mar 2021 19:45:39 -0400 Subject: [PATCH 1/2] Improve fmp4 probing and codecs support - Fallback to mp4 passthough when probing fails (not mp3 transmux) - Add 'av01' to video codecs lookup table - Improve mp4-tools codec parsing TODO and provide codec defaults for hvc1 and av01 segments Resolves #3669 --- demo/main.js | 5 +++- src/demux/transmuxer-interface.ts | 12 ++------ src/demux/transmuxer.ts | 44 +++++++++++------------------- src/remux/passthrough-remuxer.ts | 38 ++++++++++++++++++++++---- src/utils/codecs.ts | 1 + src/utils/mp4-tools.ts | 21 ++++++++------ tests/functional/auto/testbench.js | 4 ++- 7 files changed, 73 insertions(+), 52 deletions(-) diff --git a/demo/main.js b/demo/main.js index 1b69c2c1ffc..c724a72030d 100644 --- a/demo/main.js +++ b/demo/main.js @@ -807,7 +807,10 @@ function loadSelectedStream() { break; case Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR: logError( - 'Buffer add codec error for ' + data.mimeType + ':' + data.err.message + 'Buffer add codec error for ' + + data.mimeType + + ':' + + data.error.message ); break; case Hls.ErrorDetails.BUFFER_APPENDING_ERROR: diff --git a/src/demux/transmuxer-interface.ts b/src/demux/transmuxer-interface.ts index 8008e0fead5..b30ec0cfcfe 100644 --- a/src/demux/transmuxer-interface.ts +++ b/src/demux/transmuxer-interface.ts @@ -77,15 +77,9 @@ export default class TransmuxerInterface { details: ErrorDetails.INTERNAL_EXCEPTION, fatal: true, event: 'demuxerWorker', - err: { - message: - event.message + - ' (' + - event.filename + - ':' + - event.lineno + - ')', - }, + error: new Error( + `${event.message} (${event.filename}:${event.lineno})` + ), }); }; worker.postMessage({ diff --git a/src/demux/transmuxer.ts b/src/demux/transmuxer.ts index 34b31bd2b25..5b37e99942f 100644 --- a/src/demux/transmuxer.ts +++ b/src/demux/transmuxer.ts @@ -158,22 +158,12 @@ export default class Transmuxer { this.resetContiguity(); } - let { demuxer, remuxer } = this; if (this.needsProbing(uintData, discontinuity, trackSwitch)) { if (cache.dataLength) { const cachedData = cache.flush(); uintData = appendUint8Array(cachedData, uintData); } - ({ demuxer, remuxer } = this.configureTransmuxer( - uintData, - transmuxConfig - )); - } - - if (!demuxer || !remuxer) { - cache.push(uintData); - stats.executeEnd = now(); - return emptyResult(chunkMeta); + this.configureTransmuxer(uintData, transmuxConfig); } const result = this.transmux( @@ -402,7 +392,7 @@ export default class Transmuxer { private configureTransmuxer( data: Uint8Array, transmuxConfig: TransmuxConfig - ): { remuxer: Remuxer | undefined; demuxer: Demuxer | undefined } { + ) { const { config, observer, typeSupported, vendor } = this; const { audioCodec, @@ -414,35 +404,33 @@ export default class Transmuxer { // probe for content type let mux; for (let i = 0, len = muxConfig.length; i < len; i++) { - mux = muxConfig[i]; - if (mux.demux.probe(data)) { + if (muxConfig[i].demux.probe(data)) { + mux = muxConfig[i]; break; } } if (!mux) { - return { remuxer: undefined, demuxer: undefined }; + // If probing previous configs fail, use mp4 passthrough + logger.warn( + 'Failed to find demuxer by probing frag, treating as mp4 passthrough' + ); + mux = { demux: MP4Demuxer, remux: PassThroughRemuxer }; } // so let's check that current remuxer and demuxer are still valid - let demuxer = this.demuxer; - let remuxer = this.remuxer; - const Remuxer = mux.remux; - const Demuxer = mux.demux; + const demuxer = this.demuxer; + const remuxer = this.remuxer; + const Remuxer: MuxConfig['remux'] = mux.remux; + const Demuxer: MuxConfig['demux'] = mux.demux; if (!remuxer || !(remuxer instanceof Remuxer)) { - remuxer = this.remuxer = new Remuxer( - observer, - config, - typeSupported, - vendor - ); + this.remuxer = new Remuxer(observer, config, typeSupported, vendor); } if (!demuxer || !(demuxer instanceof Demuxer)) { - demuxer = this.demuxer = new Demuxer(observer, config, typeSupported); + this.demuxer = new Demuxer(observer, config, typeSupported); this.probe = Demuxer.probe; } // Ensure that muxers are always initialized with an initSegment this.resetInitSegment(initSegmentData, audioCodec, videoCodec, duration); this.resetInitialTimestamp(defaultInitPts); - return { demuxer, remuxer }; } private needsProbing( @@ -452,7 +440,7 @@ export default class Transmuxer { ): boolean { // in case of continuity change, or track switch // we might switch from content type (AAC container to TS container, or TS to fmp4 for example) - return !this.demuxer || discontinuity || trackSwitch; + return !this.demuxer || !this.remuxer || discontinuity || trackSwitch; } private getDecrypter(): Decrypter { diff --git a/src/remux/passthrough-remuxer.ts b/src/remux/passthrough-remuxer.ts index bae20691bb5..de31d60280b 100644 --- a/src/remux/passthrough-remuxer.ts +++ b/src/remux/passthrough-remuxer.ts @@ -1,10 +1,11 @@ -import type { InitData } from '../utils/mp4-tools'; +import type { InitData, InitDataTrack } from '../utils/mp4-tools'; import { getDuration, getStartDTS, offsetStartDTS, parseInitSegment, } from '../utils/mp4-tools'; +import { ElementaryStreamTypes } from '../loader/fragment'; import { logger } from '../utils/logger'; import type { TrackSet } from '../types/track'; import type { @@ -60,14 +61,19 @@ class PassThroughRemuxer implements Remuxer { } const initData = (this.initData = parseInitSegment(initSegment)); - // default audio codec if nothing specified - // TODO : extract that from initSegment + // Get codec from initSegment or fallback to default if (!audioCodec) { - audioCodec = 'mp4a.40.5'; + audioCodec = getParsedTrackCodec( + initData.audio, + ElementaryStreamTypes.AUDIO + ); } if (!videoCodec) { - videoCodec = 'avc1.42e01e'; + videoCodec = getParsedTrackCodec( + initData.video, + ElementaryStreamTypes.VIDEO + ); } const tracks: TrackSet = {}; @@ -207,4 +213,26 @@ class PassThroughRemuxer implements Remuxer { const computeInitPTS = (initData, data, timeOffset) => getStartDTS(initData, data) - timeOffset; +function getParsedTrackCodec( + track: InitDataTrack | undefined, + type: ElementaryStreamTypes.AUDIO | ElementaryStreamTypes.VIDEO +): string { + const parsedCodec = track?.codec; + if (parsedCodec && parsedCodec.length > 4) { + return parsedCodec; + } + // Since mp4-tools cannot parse full codec string (see 'TODO: Parse codec details'... in mp4-tools) + // Provide defaults based on codec type + // This allows for some playback of some fmp4 playlists without CODECS defined in manifest + if (parsedCodec === 'hvc1') { + return 'hvc1.1.c.L120.90'; + } + if (parsedCodec === 'av01') { + return 'av01.0.04M.08'; + } + if (parsedCodec === 'avc1' || type === ElementaryStreamTypes.VIDEO) { + return 'avc1.42e01e'; + } + return 'mp4a.40.5'; +} export default PassThroughRemuxer; diff --git a/src/utils/codecs.ts b/src/utils/codecs.ts index c957fee7c72..6dadd4261a5 100644 --- a/src/utils/codecs.ts +++ b/src/utils/codecs.ts @@ -40,6 +40,7 @@ const sampleEntryCodesISO = { avc3: true, avc4: true, avcp: true, + av01: true, drac: true, dvav: true, dvhe: true, diff --git a/src/utils/mp4-tools.ts b/src/utils/mp4-tools.ts index 37119db36fd..42a238d8b54 100644 --- a/src/utils/mp4-tools.ts +++ b/src/utils/mp4-tools.ts @@ -10,8 +10,8 @@ type Mp4BoxData = { const UINT32_MAX = Math.pow(2, 32) - 1; const push = [].push; -export function bin2str(buffer: Uint8Array): string { - return String.fromCharCode.apply(null, buffer); +export function bin2str(data: Uint8Array): string { + return String.fromCharCode.apply(null, data); } export function readUint16( @@ -230,7 +230,7 @@ export function parseSegmentIndex(initSegment: Uint8Array): SidxInfo | null { * the init segment is malformed. */ -interface InitDataTrack { +export interface InitDataTrack { timescale: number; id: number; codec: string; @@ -278,14 +278,19 @@ export function parseInitSegment(initSegment: Uint8Array): InitData { vide: ElementaryStreamTypes.VIDEO, }[hdlrType]; if (type) { - // TODO: Parse codec details to be able to build MIME type. - const codexBoxes = findBox(trak, ['mdia', 'minf', 'stbl', 'stsd']); + // Parse codec details + const stsd = findBox(trak, ['mdia', 'minf', 'stbl', 'stsd'])[0]; let codec; - if (codexBoxes.length) { - const codecBox = codexBoxes[0]; + if (stsd) { codec = bin2str( - codecBox.data.subarray(codecBox.start + 12, codecBox.start + 16) + stsd.data.subarray(stsd.start + 12, stsd.start + 16) ); + // TODO: Parse codec details to be able to build MIME type. + // stsd.start += 8; + // const codecBox = findBox(stsd, [codec])[0]; + // if (codecBox) { + // TODO: Codec parsing support for avc1, mp4a, hevc, av01... + // } } result[trackId] = { timescale, type }; result[type] = { timescale, id: trackId, codec }; diff --git a/tests/functional/auto/testbench.js b/tests/functional/auto/testbench.js index 36e899416f5..f397d216517 100644 --- a/tests/functional/auto/testbench.js +++ b/tests/functional/auto/testbench.js @@ -135,7 +135,9 @@ function startStream(streamUrl, config, callback, autoplay) { if (data.details === Hls.ErrorDetails.INTERNAL_EXCEPTION) { console.log('[test] > exception in :' + data.event); console.log( - data.err.stack ? JSON.stringify(data.err.stack) : data.err.message + data.error.stack + ? JSON.stringify(data.error.stack) + : data.error.message ); } callback({ code: data.details, logs: logString }); From bdafd254161693430a921919afc27890f982afef Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Tue, 23 Mar 2021 18:23:28 -0400 Subject: [PATCH 2/2] Improve handling of SourceBuffer creation errors --- MIGRATING.md | 4 ++++ api-extractor/report/hls.js.api.md | 2 ++ docs/API.md | 4 +++- docs/design.md | 1 + src/controller/buffer-controller.ts | 19 ++++++++++++++++--- src/errors.ts | 4 +++- src/utils/xhr-loader.ts | 11 ++++++++--- tests/unit/controller/buffer-controller.js | 16 +++++++++++++++- 8 files changed, 52 insertions(+), 9 deletions(-) diff --git a/MIGRATING.md b/MIGRATING.md index 183ab80660d..08b531c3dea 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -53,6 +53,10 @@ Event order and content have changed in some places. See **Breaking Changes** be - `SUBTITLE_LOAD_ERROR` - `SUBTITLE_TRACK_LOAD_TIMEOUT` - `UNKNOWN` +- Added additional error detail for streams that cannot start because source buffer(s) could not be created after parsing media codecs + - `BUFFER_INCOMPATIBLE_CODECS_ERROR` will fire instead of `BUFFER_CREATED` with an empty `tracks` list. This media error + is fatal and not recoverable. If you encounter this error make sure you include the correct CODECS string in + your manifest, as this is most likely to occur when attempting to play a fragmented mp4 playlist with unknown codecs. ### Fragment Stats diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 28a5bd92ebe..84d1073d08a 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -383,6 +383,8 @@ export enum ErrorDetails { // (undocumented) BUFFER_FULL_ERROR = "bufferFullError", // (undocumented) + BUFFER_INCOMPATIBLE_CODECS_ERROR = "bufferIncompatibleCodecsError", + // (undocumented) BUFFER_NUDGE_ON_STALL = "bufferNudgeOnStall", // (undocumented) BUFFER_SEEK_OVER_HOLE = "bufferSeekOverHole", diff --git a/docs/API.md b/docs/API.md index 316dbf0306b..4fc972ec70d 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1540,7 +1540,9 @@ Full list of errors is described below: - `Hls.ErrorDetails.FRAG_PARSING_ERROR` - raised when fragment parsing fails - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.FRAG_PARSING_ERROR`, fatal : `true` or `false`, reason : failure reason } - `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR` - raised when MediaSource fails to add new sourceBuffer - - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR`, fatal : `false`, err : error raised by MediaSource, mimeType: mimeType on which the failure happened } + - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR`, fatal : `false`, error : error raised by MediaSource, mimeType: mimeType on which the failure happened } +- `Hls.ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR` - raised when no MediaSource(s) could be created based on track codec(s) + - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR`, fatal : `true`, reason : failure reason } - `Hls.ErrorDetails.BUFFER_APPEND_ERROR` - raised when exception is raised while calling buffer append - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_APPEND_ERROR`, fatal : `true` or `false`, parent : parent stream controller } - `Hls.ErrorDetails.BUFFER_APPENDING_ERROR` - raised when exception is raised during buffer appending diff --git a/docs/design.md b/docs/design.md index 118a30c027b..80d958b356e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -287,6 +287,7 @@ design idea is pretty simple : - if auto level switch is enabled and loaded frag level is greater than 0, this error is not fatal: in that case [src/controller/level-controller.ts][] will trigger an emergency switch down to level 0. - if frag level is 0 or auto level switch is disabled, this error is marked as fatal and a call to `hls.startLoad()` could help recover it. - `BUFFER_ADD_CODEC_ERROR` is raised by [src/controller/buffer-controller.ts][] when an exception is raised when calling mediaSource.addSourceBuffer(). this error is non fatal. +- `BUFFER_INCOMPATIBLE_CODECS_ERROR` is raised by [src/controller/buffer-controller.ts][] when an exception is raised when all attempts to add SourceBuffer(s) failed. this error is fatal. - `BUFFER_APPEND_ERROR` is raised by [src/controller/buffer-controller.ts][] when an exception is raised when calling sourceBuffer.appendBuffer(). this error is non fatal and become fatal after config.appendErrorMaxRetry retries. when fatal, a call to `hls.recoverMediaError()` could help recover it. - `BUFFER_APPENDING_ERROR` is raised by [src/controller/buffer-controller.ts][] after SourceBuffer appending error. this error is fatal and a call to `hls.recoverMediaError()` could help recover it. - `BUFFER_FULL_ERROR` is raised by [src/controller/buffer-controller.ts][] if sourcebuffer is full diff --git a/src/controller/buffer-controller.ts b/src/controller/buffer-controller.ts index 533accb6b79..71d0b47b022 100644 --- a/src/controller/buffer-controller.ts +++ b/src/controller/buffer-controller.ts @@ -659,7 +659,17 @@ export default class BufferController implements ComponentAPI { this.createSourceBuffers(pendingTracks); this.pendingTracks = {}; // append any pending segments now ! - Object.keys(this.sourceBuffer).forEach((type: SourceBufferName) => { + const buffers = Object.keys(this.sourceBuffer); + if (buffers.length === 0) { + this.hls.trigger(Events.ERROR, { + type: ErrorTypes.MEDIA_ERROR, + details: ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR, + fatal: true, + reason: 'could not create source buffer for media codec(s)', + }); + return; + } + buffers.forEach((type: SourceBufferName) => { operationQueue.executeNext(type); }); } @@ -670,7 +680,7 @@ export default class BufferController implements ComponentAPI { if (!mediaSource) { throw Error('createSourceBuffers called when mediaSource was null'); } - + let tracksCreated = 0; for (const trackName in tracks) { if (!sourceBuffer[trackName]) { const track = tracks[trackName as keyof TrackSet]; @@ -698,6 +708,7 @@ export default class BufferController implements ComponentAPI { levelCodec: track.levelCodec, id: track.id, }; + tracksCreated++; } catch (err) { logger.error( `[buffer-controller]: error while trying to add sourceBuffer: ${err.message}` @@ -712,7 +723,9 @@ export default class BufferController implements ComponentAPI { } } } - this.hls.trigger(Events.BUFFER_CREATED, { tracks: this.tracks }); + if (tracksCreated) { + this.hls.trigger(Events.BUFFER_CREATED, { tracks: this.tracks }); + } } // Keep as arrow functions so that we can directly reference these functions directly as event listeners diff --git a/src/errors.ts b/src/errors.ts index d9e70b7b2d7..7552f0bd0f1 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -60,8 +60,10 @@ export enum ErrorDetails { KEY_LOAD_ERROR = 'keyLoadError', // Identifier for decrypt key load timeout error - data: { frag : fragment object} KEY_LOAD_TIMEOUT = 'keyLoadTimeOut', - // Triggered when an exception occurs while adding a sourceBuffer to MediaSource - data : { err : exception , mimeType : mimeType } + // Triggered when an exception occurs while adding a sourceBuffer to MediaSource - data : { error : exception , mimeType : mimeType } BUFFER_ADD_CODEC_ERROR = 'bufferAddCodecError', + // Triggered when source buffer(s) could not be created using level (manifest CODECS attribute), parsed media, or best guess codec(s) - data: { reason : error reason } + BUFFER_INCOMPATIBLE_CODECS_ERROR = 'bufferIncompatibleCodecsError', // Identifier for a buffer append error - data: append error description BUFFER_APPEND_ERROR = 'bufferAppendError', // Identifier for a buffer appending error event - data: appending error description diff --git a/src/utils/xhr-loader.ts b/src/utils/xhr-loader.ts index d6b1c26c8f0..530b60243dc 100644 --- a/src/utils/xhr-loader.ts +++ b/src/utils/xhr-loader.ts @@ -170,17 +170,22 @@ class XhrLoader implements Loader { } stats.loaded = stats.total = len; - const onProgress = this.callbacks!.onProgress; + if (!this.callbacks) { + return; + } + const onProgress = this.callbacks.onProgress; if (onProgress) { onProgress(stats, context, data, xhr); } - + if (!this.callbacks) { + return; + } const response = { url: xhr.responseURL, data: data, }; - this.callbacks!.onSuccess(response, stats, context, xhr); + this.callbacks.onSuccess(response, stats, context, xhr); } else { // if max nb of retries reached or if http status between 400 and 499 (such error cannot be recovered, retrying is useless), return error if ( diff --git a/tests/unit/controller/buffer-controller.js b/tests/unit/controller/buffer-controller.js index 071ce2326d1..f8f42dc2436 100644 --- a/tests/unit/controller/buffer-controller.js +++ b/tests/unit/controller/buffer-controller.js @@ -169,6 +169,7 @@ describe('BufferController tests', function () { bufferController.onMediaAttaching(Events.MEDIA_ATTACHING, { media: video, }); + sandbox.stub(bufferController.mediaSource, 'addSourceBuffer'); hls.on(Hls.Events.BUFFER_CREATED, (event, data) => { const tracks = data.tracks; @@ -177,7 +178,20 @@ describe('BufferController tests', function () { done(); }); - bufferController.pendingTracks = { video: { codec: 'testing' } }; + hls.once(Hls.Events.ERROR, (event, data) => { + // Async timeout prevents assertion from throwing in event handler + self.setTimeout(() => { + expect(data.error.message).to.equal(null); + done(); + }); + }); + + bufferController.pendingTracks = { + video: { + container: 'video/mp4', + codec: 'avc1.42e01e', + }, + }; bufferController.checkPendingTracks(); video = null;