Add support for emsg ID3 metadata in fmp4 segments - Closes #2360

src/demux/mp4demuxer.ts

@@ -14,11 +14,14 @@ import {
   findBox,
   segmentValidRange,
   appendUint8Array,
+  parseEmsg,
 } from '../utils/mp4-tools';
 import { dummyTrack } from './dummy-demuxed-track';
 import type { HlsEventEmitter } from '../events';
 import type { HlsConfig } from '../config';
 
+const emsgSchemePattern = /\/emsg[-/]ID3/i;
+
 class MP4Demuxer implements Demuxer {
   static readonly minProbeByteLength = 1024;
   private remainderData: Uint8Array | null = null;
@@ -43,7 +46,7 @@ class MP4Demuxer implements Demuxer {
     );
   }
 
-  demux(data): DemuxerResult {
+  demux(data: Uint8Array, timeOffset: number): DemuxerResult {
     // Load all data into the avc track. The CMAF remuxer will look for the data in the samples object; the rest of the fields do not matter
     let avcSamples = data;
     const avcTrack = dummyTrack() as PassthroughVideoTrack;
@@ -61,10 +64,31 @@ class MP4Demuxer implements Demuxer {
       avcTrack.samples = avcSamples;
     }
 
+    const id3Track = dummyTrack() as DemuxedMetadataTrack;
+    const emsgs = findBox(avcTrack.samples, ['emsg']);
+    if (emsgs) {
+      id3Track.inputTimeScale = 1;
+      emsgs.forEach(({ data, start, end }) => {
+        const emsgInfo = parseEmsg(data.subarray(start, end));
+        if (emsgSchemePattern.test(emsgInfo.schemeIdUri)) {
+          const pts = Number.isFinite(emsgInfo.presentationTime)
+            ? emsgInfo.presentationTime! / emsgInfo.timeScale
+            : timeOffset + emsgInfo.presentationTimeDelta! / emsgInfo.timeScale;
+          const payload = emsgInfo.payload;
+          id3Track.samples.push({
+            data: payload,
+            len: payload.byteLength,
+            dts: pts,
+            pts: pts,
+          });
+        }
+      });
+    }
+
     return {
       audioTrack: dummyTrack() as DemuxedAudioTrack,
       avcTrack,
-      id3Track: dummyTrack() as DemuxedMetadataTrack,
+      id3Track,
       textTrack: dummyTrack() as DemuxedUserdataTrack,
     };
   }

src/remux/mp4-remuxer.ts

@@ -244,11 +244,20 @@ export default class MP4Remuxer implements Remuxer {
     // Allow ID3 and text to remux, even if more audio/video samples are required
     if (this.ISGenerated) {
       if (id3Track.samples.length) {
-        id3 = this.remuxID3(id3Track, timeOffset);
+        id3 = flushTextTrackMetadataCueSamples(
+          id3Track,
+          timeOffset,
+          this._initPTS,
+          this._initDTS
+        );
       }
 
       if (textTrack.samples.length) {
-        text = this.remuxText(textTrack, timeOffset);
+        text = flushTextTrackUserdataCueSamples(
+          textTrack,
+          timeOffset,
+          this._initPTS
+        );
       }
     }
 
@@ -971,62 +980,6 @@ export default class MP4Remuxer implements Remuxer {
 
     return this.remuxAudio(track, timeOffset, contiguous, false);
   }
-
-  remuxID3(
-    track: DemuxedMetadataTrack,
-    timeOffset: number
-  ): RemuxedMetadata | undefined {
-    const length = track.samples.length;
-    if (!length) {
-      return;
-    }
-    const inputTimeScale = track.inputTimeScale;
-    const initPTS = this._initPTS;
-    const initDTS = this._initDTS;
-    for (let index = 0; index < length; index++) {
-      const sample = track.samples[index];
-      // setting id3 pts, dts to relative time
-      // using this._initPTS and this._initDTS to calculate relative time
-      sample.pts =
-        normalizePts(sample.pts - initPTS, timeOffset * inputTimeScale) /
-        inputTimeScale;
-      sample.dts =
-        normalizePts(sample.dts - initDTS, timeOffset * inputTimeScale) /
-        inputTimeScale;
-    }
-    const samples = track.samples;
-    track.samples = [];
-    return {
-      samples,
-    };
-  }
-
-  remuxText(
-    track: DemuxedUserdataTrack,
-    timeOffset: number
-  ): RemuxedUserdata | undefined {
-    const length = track.samples.length;
-    if (!length) {
-      return;
-    }
-
-    const inputTimeScale = track.inputTimeScale;
-    const initPTS = this._initPTS;
-    for (let index = 0; index < length; index++) {
-      const sample = track.samples[index];
-      // setting text pts, dts to relative time
-      // using this._initPTS and this._initDTS to calculate relative time
-      sample.pts =
-        normalizePts(sample.pts - initPTS, timeOffset * inputTimeScale) /
-        inputTimeScale;
-    }
-    track.samples.sort((a, b) => a.pts - b.pts);
-    const samples = track.samples;
-    track.samples = [];
-    return {
-      samples,
-    };
-  }
 }
 
 export function normalizePts(value: number, reference: number | null): number {
@@ -1061,6 +1014,62 @@ function findKeyframeIndex(samples: Array<AvcSample>): number {
   return -1;
 }
 
+export function flushTextTrackMetadataCueSamples(
+  track: DemuxedMetadataTrack,
+  timeOffset: number,
+  initPTS: number,
+  initDTS: number
+): RemuxedMetadata | undefined {
+  const length = track.samples.length;
+  if (!length) {
+    return;
+  }
+  const inputTimeScale = track.inputTimeScale;
+  for (let index = 0; index < length; index++) {
+    const sample = track.samples[index];
+    // setting id3 pts, dts to relative time
+    // using this._initPTS and this._initDTS to calculate relative time
+    sample.pts =
+      normalizePts(sample.pts - initPTS, timeOffset * inputTimeScale) /
+      inputTimeScale;
+    sample.dts =
+      normalizePts(sample.dts - initDTS, timeOffset * inputTimeScale) /
+      inputTimeScale;
+  }
+  const samples = track.samples;
+  track.samples = [];
+  return {
+    samples,
+  };
+}
+
+export function flushTextTrackUserdataCueSamples(
+  track: DemuxedUserdataTrack,
+  timeOffset: number,
+  initPTS: number
+): RemuxedUserdata | undefined {
+  const length = track.samples.length;
+  if (!length) {
+    return;
+  }
+
+  const inputTimeScale = track.inputTimeScale;
+  for (let index = 0; index < length; index++) {
+    const sample = track.samples[index];
+    // setting text pts, dts to relative time
+    // using this._initPTS and this._initDTS to calculate relative time
+    sample.pts =
+      normalizePts(sample.pts - initPTS, timeOffset * inputTimeScale) /
+      inputTimeScale;
+  }
+  track.samples.sort((a, b) => a.pts - b.pts);
+  const samples = track.samples;
+  track.samples = [];
+  return {
+    samples,
+  };
+}
+
 class Mp4Sample {
   public size: number;
   public duration: number;

src/remux/passthrough-remuxer.ts

@@ -1,3 +1,4 @@
+import { flushTextTrackMetadataCueSamples } from './mp4-remuxer';
 import type { InitData, InitDataTrack } from '../utils/mp4-tools';
 import {
   getDuration,
@@ -201,9 +202,14 @@ class PassThroughRemuxer implements Remuxer {
 
     result.audio = track.type === 'audio' ? track : undefined;
     result.video = track.type !== 'audio' ? track : undefined;
-    result.text = textTrack;
-    result.id3 = id3Track;
     result.initSegment = initSegment;
+    const id3InitPts = this.initPTS ?? 0;
+    result.id3 = flushTextTrackMetadataCueSamples(
+      id3Track,
+      timeOffset,
+      id3InitPts,
+      id3InitPts
+    );
 
     return result;
   }

src/utils/mp4-tools.ts

@@ -585,3 +585,99 @@ export function appendUint8Array(
 
   return temp;
 }
+
+export interface IEmsgParsingData {
+  schemeIdUri: string;
+  value: string;
+  timeScale: number;
+  presentationTimeDelta?: number;
+  presentationTime?: number;
+  eventDuration: number;
+  id: number;
+  payload: Uint8Array;
+}
+
+export function parseEmsg(data: Uint8Array): IEmsgParsingData {
+  const version = data[0];
+  let schemeIdUri: string = '';
+  let value: string = '';
+  let timeScale: number;
+  let presentationTimeDelta: number = 0;
+  let presentationTime: number = 0;
+  let eventDuration: number;
+  let id: number;
+  let offset: number = 0;
+
+  if (version === 0) {
+    while (bin2str(data.subarray(offset, offset + 1)) !== '\0') {
+      schemeIdUri += bin2str(data.subarray(offset, offset + 1));
+      offset += 1;
+    }
+
+    schemeIdUri += bin2str(data.subarray(offset, offset + 1));
+    offset += 1;
+
+    while (bin2str(data.subarray(offset, offset + 1)) !== '\0') {
+      value += bin2str(data.subarray(offset, offset + 1));
+      offset += 1;
+    }
+
+    value += bin2str(data.subarray(offset, offset + 1));
+    offset += 1;
+
+    timeScale = readUint32(data, 12);
+    presentationTimeDelta = readUint32(data, 16);
+    eventDuration = readUint32(data, 20);
+    id = readUint32(data, 24);
+    offset = 28;
+  } else {
+    offset += 4;
+    timeScale = readUint32(data, offset);
+    offset += 4;
+    const leftPresentationTime = readUint32(data, offset);
+    offset += 4;
+    const rightPresentationTime = readUint32(data, offset);
+    offset += 4;
+    presentationTime = 2 ** 32 * leftPresentationTime + rightPresentationTime;
+    if (!Number.isSafeInteger(presentationTime)) {
+      presentationTime = Number.MAX_SAFE_INTEGER;
+      // eslint-disable-next-line no-console
+      console.warn(
+        'Presentation time exceeds safe integer limit and wrapped to max safe integer in parsing emsg box'
+      );
+    }
+
+    eventDuration = readUint32(data, offset);
+    offset += 4;
+    id = readUint32(data, offset);
+    offset += 4;
+
+    while (bin2str(data.subarray(offset, offset + 1)) !== '\0') {
+      schemeIdUri += bin2str(data.subarray(offset, offset + 1));
+      offset += 1;
+    }
+
+    schemeIdUri += bin2str(data.subarray(offset, offset + 1));
+    offset += 1;
+
+    while (bin2str(data.subarray(offset, offset + 1)) !== '\0') {
+      value += bin2str(data.subarray(offset, offset + 1));
+      offset += 1;
+    }
+
+    value += bin2str(data.subarray(offset, offset + 1));
+    offset += 1;
+  }
+  const payload = data.subarray(offset, data.byteLength);
+
+  return {
+    schemeIdUri,
+    value,
+    timeScale,
+    presentationTime,
+    presentationTimeDelta,
+    eventDuration,
+    id,
+    payload,
+  };
+}