Merge pull request #3672 from video-dev/bugfix/fmp4-probing

Improve handling of fmp4 streams when probing fails or codecs are unknown

MIGRATING.md

@@ -53,6 +53,10 @@ Event order and content have changed in some places. See **Breaking Changes** be
   - `SUBTITLE_LOAD_ERROR`
   - `SUBTITLE_TRACK_LOAD_TIMEOUT`
   - `UNKNOWN`
+- Added additional error detail for streams that cannot start because source buffer(s) could not be created after parsing media codecs
+  - `BUFFER_INCOMPATIBLE_CODECS_ERROR` will fire instead of `BUFFER_CREATED` with an empty `tracks` list. This media error
+    is fatal and not recoverable. If you encounter this error make sure you include the correct CODECS string in
+    your manifest, as this is most likely to occur when attempting to play a fragmented mp4 playlist with unknown codecs.
 
 ### Fragment Stats
 

api-extractor/report/hls.js.api.md

@@ -383,6 +383,8 @@ export enum ErrorDetails {
     // (undocumented)
     BUFFER_FULL_ERROR = "bufferFullError",
     // (undocumented)
+    BUFFER_INCOMPATIBLE_CODECS_ERROR = "bufferIncompatibleCodecsError",
+    // (undocumented)
     BUFFER_NUDGE_ON_STALL = "bufferNudgeOnStall",
     // (undocumented)
     BUFFER_SEEK_OVER_HOLE = "bufferSeekOverHole",

demo/main.js

@@ -807,7 +807,10 @@ function loadSelectedStream() {
         break;
       case Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR:
         logError(
-          'Buffer add codec error for ' + data.mimeType + ':' + data.err.message
+          'Buffer add codec error for ' +
+            data.mimeType +
+            ':' +
+            data.error.message
         );
         break;
       case Hls.ErrorDetails.BUFFER_APPENDING_ERROR:

docs/API.md

@@ -1540,7 +1540,9 @@ Full list of errors is described below:
 - `Hls.ErrorDetails.FRAG_PARSING_ERROR` - raised when fragment parsing fails
   - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.FRAG_PARSING_ERROR`, fatal : `true` or `false`, reason : failure reason }
 - `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR` - raised when MediaSource fails to add new sourceBuffer
-  - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR`, fatal : `false`, err : error raised by MediaSource, mimeType: mimeType on which the failure happened }
+  - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR`, fatal : `false`, error : error raised by MediaSource, mimeType: mimeType on which the failure happened }
+- `Hls.ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR` - raised when no MediaSource(s) could be created based on track codec(s)
+  - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR`, fatal : `true`, reason : failure reason }
 - `Hls.ErrorDetails.BUFFER_APPEND_ERROR` - raised when exception is raised while calling buffer append
   - data: { type : `MEDIA_ERROR`, details : `Hls.ErrorDetails.BUFFER_APPEND_ERROR`, fatal : `true` or `false`, parent : parent stream controller }
 - `Hls.ErrorDetails.BUFFER_APPENDING_ERROR` - raised when exception is raised during buffer appending

docs/design.md

@@ -287,6 +287,7 @@ design idea is pretty simple :
   - if auto level switch is enabled and loaded frag level is greater than 0, this error is not fatal: in that case [src/controller/level-controller.ts][] will trigger an emergency switch down to level 0.
   - if frag level is 0 or auto level switch is disabled, this error is marked as fatal and a call to `hls.startLoad()` could help recover it.
 - `BUFFER_ADD_CODEC_ERROR` is raised by [src/controller/buffer-controller.ts][] when an exception is raised when calling mediaSource.addSourceBuffer(). this error is non fatal.
+- `BUFFER_INCOMPATIBLE_CODECS_ERROR` is raised by [src/controller/buffer-controller.ts][] when an exception is raised when all attempts to add SourceBuffer(s) failed. this error is fatal.
 - `BUFFER_APPEND_ERROR` is raised by [src/controller/buffer-controller.ts][] when an exception is raised when calling sourceBuffer.appendBuffer(). this error is non fatal and become fatal after config.appendErrorMaxRetry retries. when fatal, a call to `hls.recoverMediaError()` could help recover it.
 - `BUFFER_APPENDING_ERROR` is raised by [src/controller/buffer-controller.ts][] after SourceBuffer appending error. this error is fatal and a call to `hls.recoverMediaError()` could help recover it.
 - `BUFFER_FULL_ERROR` is raised by [src/controller/buffer-controller.ts][] if sourcebuffer is full

src/controller/buffer-controller.ts

@@ -659,7 +659,17 @@ export default class BufferController implements ComponentAPI {
       this.createSourceBuffers(pendingTracks);
       this.pendingTracks = {};
       // append any pending segments now !
-      Object.keys(this.sourceBuffer).forEach((type: SourceBufferName) => {
+      const buffers = Object.keys(this.sourceBuffer);
+      if (buffers.length === 0) {
+        this.hls.trigger(Events.ERROR, {
+          type: ErrorTypes.MEDIA_ERROR,
+          details: ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR,
+          fatal: true,
+          reason: 'could not create source buffer for media codec(s)',
+        });
+        return;
+      }
+      buffers.forEach((type: SourceBufferName) => {
         operationQueue.executeNext(type);
       });
     }
@@ -670,7 +680,7 @@ export default class BufferController implements ComponentAPI {
     if (!mediaSource) {
       throw Error('createSourceBuffers called when mediaSource was null');
     }
-
+    let tracksCreated = 0;
     for (const trackName in tracks) {
       if (!sourceBuffer[trackName]) {
         const track = tracks[trackName as keyof TrackSet];
@@ -698,6 +708,7 @@ export default class BufferController implements ComponentAPI {
             levelCodec: track.levelCodec,
             id: track.id,
           };
+          tracksCreated++;
         } catch (err) {
           logger.error(
             `[buffer-controller]: error while trying to add sourceBuffer: ${err.message}`
@@ -712,7 +723,9 @@ export default class BufferController implements ComponentAPI {
         }
       }
     }
-    this.hls.trigger(Events.BUFFER_CREATED, { tracks: this.tracks });
+    if (tracksCreated) {
+      this.hls.trigger(Events.BUFFER_CREATED, { tracks: this.tracks });
+    }
   }
 
   // Keep as arrow functions so that we can directly reference these functions directly as event listeners

src/demux/transmuxer-interface.ts

@@ -77,15 +77,9 @@ export default class TransmuxerInterface {
             details: ErrorDetails.INTERNAL_EXCEPTION,
             fatal: true,
             event: 'demuxerWorker',
-            err: {
-              message:
-                event.message +
-                ' (' +
-                event.filename +
-                ':' +
-                event.lineno +
-                ')',
-            },
+            error: new Error(
+              `${event.message}  (${event.filename}:${event.lineno})`
+            ),
           });
         };
         worker.postMessage({

src/demux/transmuxer.ts

@@ -158,22 +158,12 @@ export default class Transmuxer {
       this.resetContiguity();
     }
 
-    let { demuxer, remuxer } = this;
     if (this.needsProbing(uintData, discontinuity, trackSwitch)) {
       if (cache.dataLength) {
         const cachedData = cache.flush();
         uintData = appendUint8Array(cachedData, uintData);
       }
-      ({ demuxer, remuxer } = this.configureTransmuxer(
-        uintData,
-        transmuxConfig
-      ));
-    }
-
-    if (!demuxer || !remuxer) {
-      cache.push(uintData);
-      stats.executeEnd = now();
-      return emptyResult(chunkMeta);
+      this.configureTransmuxer(uintData, transmuxConfig);
     }
 
     const result = this.transmux(
@@ -402,7 +392,7 @@ export default class Transmuxer {
   private configureTransmuxer(
     data: Uint8Array,
     transmuxConfig: TransmuxConfig
-  ): { remuxer: Remuxer | undefined; demuxer: Demuxer | undefined } {
+  ) {
     const { config, observer, typeSupported, vendor } = this;
     const {
       audioCodec,
@@ -414,35 +404,33 @@ export default class Transmuxer {
     // probe for content type
     let mux;
     for (let i = 0, len = muxConfig.length; i < len; i++) {
-      mux = muxConfig[i];
-      if (mux.demux.probe(data)) {
+      if (muxConfig[i].demux.probe(data)) {
+        mux = muxConfig[i];
         break;
       }
     }
     if (!mux) {
-      return { remuxer: undefined, demuxer: undefined };
+      // If probing previous configs fail, use mp4 passthrough
+      logger.warn(
+        'Failed to find demuxer by probing frag, treating as mp4 passthrough'
+      );
+      mux = { demux: MP4Demuxer, remux: PassThroughRemuxer };
     }
     // so let's check that current remuxer and demuxer are still valid
-    let demuxer = this.demuxer;
-    let remuxer = this.remuxer;
-    const Remuxer = mux.remux;
-    const Demuxer = mux.demux;
+    const demuxer = this.demuxer;
+    const remuxer = this.remuxer;
+    const Remuxer: MuxConfig['remux'] = mux.remux;
+    const Demuxer: MuxConfig['demux'] = mux.demux;
     if (!remuxer || !(remuxer instanceof Remuxer)) {
-      remuxer = this.remuxer = new Remuxer(
-        observer,
-        config,
-        typeSupported,
-        vendor
-      );
+      this.remuxer = new Remuxer(observer, config, typeSupported, vendor);
     }
     if (!demuxer || !(demuxer instanceof Demuxer)) {
-      demuxer = this.demuxer = new Demuxer(observer, config, typeSupported);
+      this.demuxer = new Demuxer(observer, config, typeSupported);
       this.probe = Demuxer.probe;
     }
     // Ensure that muxers are always initialized with an initSegment
     this.resetInitSegment(initSegmentData, audioCodec, videoCodec, duration);
     this.resetInitialTimestamp(defaultInitPts);
-    return { demuxer, remuxer };
   }
 
   private needsProbing(
@@ -452,7 +440,7 @@ export default class Transmuxer {
   ): boolean {
     // in case of continuity change, or track switch
     // we might switch from content type (AAC container to TS container, or TS to fmp4 for example)
-    return !this.demuxer || discontinuity || trackSwitch;
+    return !this.demuxer || !this.remuxer || discontinuity || trackSwitch;
   }
 
   private getDecrypter(): Decrypter {

src/errors.ts

@@ -60,8 +60,10 @@ export enum ErrorDetails {
   KEY_LOAD_ERROR = 'keyLoadError',
   // Identifier for decrypt key load timeout error - data: { frag : fragment object}
   KEY_LOAD_TIMEOUT = 'keyLoadTimeOut',
-  // Triggered when an exception occurs while adding a sourceBuffer to MediaSource - data : {  err : exception , mimeType : mimeType }
+  // Triggered when an exception occurs while adding a sourceBuffer to MediaSource - data : { error : exception , mimeType : mimeType }
   BUFFER_ADD_CODEC_ERROR = 'bufferAddCodecError',
+  // Triggered when source buffer(s) could not be created using level (manifest CODECS attribute), parsed media, or best guess codec(s) - data: { reason : error reason }
+  BUFFER_INCOMPATIBLE_CODECS_ERROR = 'bufferIncompatibleCodecsError',
   // Identifier for a buffer append error - data: append error description
   BUFFER_APPEND_ERROR = 'bufferAppendError',
   // Identifier for a buffer appending error event - data: appending error description

src/remux/passthrough-remuxer.ts

@@ -1,10 +1,11 @@
-import type { InitData } from '../utils/mp4-tools';
+import type { InitData, InitDataTrack } from '../utils/mp4-tools';
 import {
   getDuration,
   getStartDTS,
   offsetStartDTS,
   parseInitSegment,
 } from '../utils/mp4-tools';
+import { ElementaryStreamTypes } from '../loader/fragment';
 import { logger } from '../utils/logger';
 import type { TrackSet } from '../types/track';
 import type {
@@ -60,14 +61,19 @@ class PassThroughRemuxer implements Remuxer {
     }
     const initData = (this.initData = parseInitSegment(initSegment));
 
-    // default audio codec if nothing specified
-    // TODO : extract that from initSegment
+    // Get codec from initSegment or fallback to default
     if (!audioCodec) {
-      audioCodec = 'mp4a.40.5';
+      audioCodec = getParsedTrackCodec(
+        initData.audio,
+        ElementaryStreamTypes.AUDIO
+      );
     }
 
     if (!videoCodec) {
-      videoCodec = 'avc1.42e01e';
+      videoCodec = getParsedTrackCodec(
+        initData.video,
+        ElementaryStreamTypes.VIDEO
+      );
     }
 
     const tracks: TrackSet = {};
@@ -207,4 +213,26 @@ class PassThroughRemuxer implements Remuxer {
 const computeInitPTS = (initData, data, timeOffset) =>
   getStartDTS(initData, data) - timeOffset;
 
+function getParsedTrackCodec(
+  track: InitDataTrack | undefined,
+  type: ElementaryStreamTypes.AUDIO | ElementaryStreamTypes.VIDEO
+): string {
+  const parsedCodec = track?.codec;
+  if (parsedCodec && parsedCodec.length > 4) {
+    return parsedCodec;
+  }
+  // Since mp4-tools cannot parse full codec string (see 'TODO: Parse codec details'... in mp4-tools)
+  // Provide defaults based on codec type
+  // This allows for some playback of some fmp4 playlists without CODECS defined in manifest
+  if (parsedCodec === 'hvc1') {
+    return 'hvc1.1.c.L120.90';
+  }
+  if (parsedCodec === 'av01') {
+    return 'av01.0.04M.08';
+  }
+  if (parsedCodec === 'avc1' || type === ElementaryStreamTypes.VIDEO) {
+    return 'avc1.42e01e';
+  }
+  return 'mp4a.40.5';
+}
 export default PassThroughRemuxer;

src/utils/codecs.ts

@@ -40,6 +40,7 @@ const sampleEntryCodesISO = {
     avc3: true,
     avc4: true,
     avcp: true,
+    av01: true,
     drac: true,
     dvav: true,
     dvhe: true,

src/utils/mp4-tools.ts

@@ -10,8 +10,8 @@ type Mp4BoxData = {
 const UINT32_MAX = Math.pow(2, 32) - 1;
 const push = [].push;
 
-export function bin2str(buffer: Uint8Array): string {
-  return String.fromCharCode.apply(null, buffer);
+export function bin2str(data: Uint8Array): string {
+  return String.fromCharCode.apply(null, data);
 }
 
 export function readUint16(
@@ -230,7 +230,7 @@ export function parseSegmentIndex(initSegment: Uint8Array): SidxInfo | null {
  * the init segment is malformed.
  */
 
-interface InitDataTrack {
+export interface InitDataTrack {
   timescale: number;
   id: number;
   codec: string;
@@ -278,14 +278,19 @@ export function parseInitSegment(initSegment: Uint8Array): InitData {
             vide: ElementaryStreamTypes.VIDEO,
           }[hdlrType];
           if (type) {
-            // TODO: Parse codec details to be able to build MIME type.
-            const codexBoxes = findBox(trak, ['mdia', 'minf', 'stbl', 'stsd']);
+            // Parse codec details
+            const stsd = findBox(trak, ['mdia', 'minf', 'stbl', 'stsd'])[0];
             let codec;
-            if (codexBoxes.length) {
-              const codecBox = codexBoxes[0];
+            if (stsd) {
               codec = bin2str(
-                codecBox.data.subarray(codecBox.start + 12, codecBox.start + 16)
+                stsd.data.subarray(stsd.start + 12, stsd.start + 16)
               );
+              // TODO: Parse codec details to be able to build MIME type.
+              // stsd.start += 8;
+              // const codecBox = findBox(stsd, [codec])[0];
+              // if (codecBox) {
+              //   TODO: Codec parsing support for avc1, mp4a, hevc, av01...
+              // }
             }
             result[trackId] = { timescale, type };
             result[type] = { timescale, id: trackId, codec };

src/utils/xhr-loader.ts

@@ -170,17 +170,22 @@ class XhrLoader implements Loader<LoaderContext> {
           }
           stats.loaded = stats.total = len;
 
-          const onProgress = this.callbacks!.onProgress;
+          if (!this.callbacks) {
+            return;
+          }
+          const onProgress = this.callbacks.onProgress;
           if (onProgress) {
             onProgress(stats, context, data, xhr);
           }
-
+          if (!this.callbacks) {
+            return;
+          }
           const response = {
             url: xhr.responseURL,
             data: data,
           };
 
-          this.callbacks!.onSuccess(response, stats, context, xhr);
+          this.callbacks.onSuccess(response, stats, context, xhr);
         } else {
           // if max nb of retries reached or if http status between 400 and 499 (such error cannot be recovered, retrying is useless), return error
           if (

tests/functional/auto/testbench.js

@@ -135,7 +135,9 @@ function startStream(streamUrl, config, callback, autoplay) {
         if (data.details === Hls.ErrorDetails.INTERNAL_EXCEPTION) {
           console.log('[test] > exception in :' + data.event);
           console.log(
-            data.err.stack ? JSON.stringify(data.err.stack) : data.err.message
+            data.error.stack
+              ? JSON.stringify(data.error.stack)
+              : data.error.message
           );
         }
         callback({ code: data.details, logs: logString });