Skip to content

Commit

Permalink
Remux overlapping AAC samples so that they are appended over earlier …
Browse files Browse the repository at this point in the history 
…samples segment rather than dropping them
  • Loading branch information
Rob Walch committed May 16, 2021
1 parent ed9bb15 commit 74ac100
Showing 1 changed file with 17 additions and 79 deletions.
96 changes: 17 additions & 79 deletions src/remux/mp4-remuxer.ts
View file
Expand Up @@ -747,8 +747,9 @@ export default class MP4Remuxer implements Remuxer {
// frame.

if (track.isAAC) {
const alignedWithVideo = videoTimeOffset !== undefined;
const maxAudioFramesDrift = this.config.maxAudioFramesDrift;
for (let i = 0, nextPts = nextAudioPts; i < inputSamples.length; ) {
for (let i = 0, nextPts = nextAudioPts; i < inputSamples.length; i++) {
// First, let's see how far off this frame is from where we expect it to be
const sample = inputSamples[i];
const pts = sample.pts;
Expand All @@ -758,29 +759,19 @@ export default class MP4Remuxer implements Remuxer {
// When remuxing with video, if we're overlapping by more than a duration, drop this sample to stay in sync
if (
delta <= -maxAudioFramesDrift * inputSampleDuration &&
videoTimeOffset !== undefined
alignedWithVideo
) {
if (contiguous || i > 0) {
logger.warn(
`[mp4-remuxer]: Dropping 1 audio frame @ ${(
nextPts / inputTimeScale
).toFixed(3)}s due to ${Math.round(duration)} ms overlap.`
);
inputSamples.splice(i, 1);
// Don't touch nextPtsNorm or i
} else {
// When changing qualities we can't trust that audio has been appended up to nextAudioPts
// Warn about the overlap but do not drop samples as that can introduce buffer gaps
if (i === 0) {
logger.warn(
`Audio frame @ ${(pts / inputTimeScale).toFixed(
3
)}s overlaps nextAudioPts by ${Math.round(
(1000 * delta) / inputTimeScale
)} ms.`
);
nextPts = pts + inputSampleDuration;
i++;
this.nextAudioPts = nextAudioPts = pts;
}
nextPts = pts;
} // eslint-disable-line brace-style

// Insert missing frames if:
Expand All @@ -791,12 +782,19 @@ export default class MP4Remuxer implements Remuxer {
else if (
delta >= maxAudioFramesDrift * inputSampleDuration &&
duration < MAX_SILENT_FRAME_DURATION &&
videoTimeOffset !== undefined
alignedWithVideo
) {
const missing = Math.floor(delta / inputSampleDuration);
let missing = Math.round(delta / inputSampleDuration);
// Adjust nextPts so that silent samples are aligned with media pts. This will prevent media samples from
// later being shifted if nextPts is based on timeOffset and delta is not a multiple of inputSampleDuration.
nextPts = pts - missing * inputSampleDuration;
if (nextPts < 0) {
missing--;
nextPts += inputSampleDuration;
}
if (i === 0) {
this.nextAudioPts = nextAudioPts = nextPts;
}
logger.warn(
`[mp4-remuxer]: Injecting ${missing} audio frame @ ${(
nextPts / inputTimeScale
Expand Down Expand Up @@ -824,17 +822,9 @@ export default class MP4Remuxer implements Remuxer {
nextPts += inputSampleDuration;
i++;
}

// Adjust sample to next expected pts
sample.pts = sample.dts = nextPts;
nextPts += inputSampleDuration;
i++;
} else {
// Otherwise, just adjust pts
sample.pts = sample.dts = nextPts;
nextPts += inputSampleDuration;
i++;
}
sample.pts = sample.dts = nextPts;
nextPts += inputSampleDuration;
}
}
let firstPTS: number | null = null;
Expand All @@ -855,42 +845,7 @@ export default class MP4Remuxer implements Remuxer {
const prevSample = outputSamples[j - 1];
prevSample.duration = Math.round((pts - lastPTS) / scaleFactor);
} else {
const delta = Math.round(
(1000 * (pts - nextAudioPts)) / inputTimeScale
);
let numMissingFrames = 0;
// if fragment are contiguous, detect hole/overlapping between fragments
// contiguous fragments are consecutive fragments from same quality level (same level, new SN = old SN + 1)
if (contiguous && track.isAAC) {
if (delta > 0 && delta < MAX_SILENT_FRAME_DURATION) {
numMissingFrames = Math.round(
(pts - nextAudioPts) / inputSampleDuration
);
logger.log(
`[mp4-remuxer]: ${delta} ms hole between AAC samples detected,filling it`
);
if (numMissingFrames > 0) {
fillFrame = AAC.getSilentFrame(
track.manifestCodec || track.codec,
track.channelCount
);
if (!fillFrame) {
fillFrame = unit.subarray();
}

mdatSize += numMissingFrames * fillFrame.length;
}
// if we have frame overlap, overlapping for more than half a frame duraion
} else if (delta < -12) {
// drop overlapping audio frames... browser will deal with it
logger.log(
`[mp4-remuxer]: drop overlapping AAC sample, expected/parsed/delta:${(
nextAudioPts / inputTimeScale
).toFixed(3)}s/${(pts / inputTimeScale).toFixed(3)}s/${-delta}ms`
);
mdatSize -= unit.byteLength;
continue;
}
// set PTS/DTS to expected PTS/DTS
pts = nextAudioPts;
}
Expand Down Expand Up @@ -921,23 +876,6 @@ export default class MP4Remuxer implements Remuxer {
// no audio samples
return;
}
for (let i = 0; i < numMissingFrames; i++) {
fillFrame = AAC.getSilentFrame(
track.manifestCodec || track.codec,
track.channelCount
);
if (!fillFrame) {
logger.log(
'[mp4-remuxer]: Unable to get silent frame for given audio codec; duplicating the current frame instead'
);
fillFrame = unit.subarray();
}
mdat.set(fillFrame, offset);
offset += fillFrame.byteLength;
outputSamples.push(
new Mp4Sample(true, AAC_SAMPLES_PER_FRAME, fillFrame.byteLength, 0)
);
}
}
mdat.set(unit, offset);
const unitLen = unit.byteLength;
Expand Down

0 comments on commit 74ac100

Please sign in to comment.