From 7efc406645c184122d5ad4d558c218636966c066 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Wed, 8 Jul 2020 23:13:55 -0400 Subject: [PATCH 1/6] Fix dropped frames caused by AVC min/max PTS not matching first/last PTS #2873 --- src/remux/mp4-remuxer.js | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/remux/mp4-remuxer.js b/src/remux/mp4-remuxer.js index ad9fdd85f56..fb34ed0e9a1 100644 --- a/src/remux/mp4-remuxer.js +++ b/src/remux/mp4-remuxer.js @@ -144,7 +144,7 @@ class MP4Remuxer { }; if (computePTSDTS) { // remember first PTS of this demuxing context. for audio, PTS = DTS - initPTS = initDTS = audioSamples[0].pts - audioTrack.inputTimeScale * timeOffset; + initPTS = initDTS = audioSamples[0].pts - Math.round(audioTrack.inputTimeScale * timeOffset); } } @@ -163,8 +163,9 @@ class MP4Remuxer { } }; if (computePTSDTS) { - initPTS = Math.min(initPTS, videoSamples[0].pts - inputTimeScale * timeOffset); - initDTS = Math.min(initDTS, videoSamples[0].dts - inputTimeScale * timeOffset); + const startPTS = Math.round(inputTimeScale * timeOffset); + initPTS = Math.min(initPTS, videoSamples[0].pts - startPTS); + initDTS = Math.min(initDTS, videoSamples[0].dts - startPTS); this.observer.trigger(Event.INIT_PTS_FOUND, { initPTS }); } } else if (computePTSDTS && tracks.audio) { @@ -189,10 +190,10 @@ class MP4Remuxer { let mp4SampleDuration; let mdat; let moof; - let firstPTS; let firstDTS; - let lastPTS; let lastDTS; + let minPTS = Number.MAX_SAFE_INTEGER; + let maxPTS = -Number.MAX_SAFE_INTEGER; const timeScale = track.timescale; const inputSamples = track.samples; const outputSamples = []; @@ -232,6 +233,9 @@ class MP4Remuxer { inputSamples.forEach(function (sample) { sample.pts = ptsNormalize(sample.pts - initPTS, nextAvcDts); sample.dts = ptsNormalize(sample.dts - initPTS, nextAvcDts); + + minPTS = Math.min(sample.pts, minPTS); + maxPTS = Math.max(sample.pts, maxPTS); }); // sort video samples by DTS then PTS then demux id order @@ -250,10 +254,9 @@ class MP4Remuxer { } } - // compute first DTS and last DTS, normalize them against reference value - let sample = inputSamples[0]; - firstDTS = Math.max(sample.dts, 0); - firstPTS = Math.max(sample.pts, 0); + // Get first/last DTS + firstDTS = inputSamples[0].dts; + lastDTS = inputSamples[inputSamples.length - 1].dts; // check timestamp continuity accross consecutive fragments (this is to remove inter-fragment gap/hole) let delta = firstDTS - nextAvcDts; @@ -270,17 +273,12 @@ class MP4Remuxer { firstDTS = nextAvcDts; inputSamples[0].dts = firstDTS; // offset PTS as well, ensure that PTS is smaller or equal than new DTS - firstPTS = Math.max(firstPTS - delta, nextAvcDts); - inputSamples[0].pts = firstPTS; - logger.log(`Video: PTS/DTS adjusted: ${toMsFromMpegTsClock(firstPTS, true)}/${toMsFromMpegTsClock(firstDTS, true)}, delta: ${toMsFromMpegTsClock(delta, true)} ms`); + minPTS = Math.max(minPTS - delta, nextAvcDts); + inputSamples[0].pts = minPTS; + logger.log(`Video: PTS/DTS adjusted: ${toMsFromMpegTsClock(minPTS, true)}/${toMsFromMpegTsClock(firstDTS, true)}, delta: ${toMsFromMpegTsClock(delta, true)} ms`); } } - // compute lastPTS/lastDTS - sample = inputSamples[inputSamples.length - 1]; - lastDTS = Math.max(sample.dts, 0); - lastPTS = Math.max(sample.pts, 0, lastDTS); - // on Safari let's signal the same sample duration for all samples // sample duration (as expected by trun MP4 boxes), should be the delta between sample DTS // set this constant duration as being the avg delta between consecutive DTS. @@ -357,7 +355,7 @@ class MP4Remuxer { // the duration of the last frame to minimize any potential gap between segments. let maxBufferHole = config.maxBufferHole, gapTolerance = Math.floor(maxBufferHole * timeScale), - deltaToFrameEnd = (audioTrackLength ? firstPTS + audioTrackLength * timeScale : this.nextAudioPts) - avcSample.pts; + deltaToFrameEnd = (audioTrackLength ? minPTS + audioTrackLength * timeScale : this.nextAudioPts) - avcSample.pts; if (deltaToFrameEnd > gapTolerance) { // We subtract lastFrameDuration from deltaToFrameEnd to try to prevent any video // frame overlap. maxBufferHole should be >> lastFrameDuration anyway. @@ -414,8 +412,8 @@ class MP4Remuxer { let data = { data1: moof, data2: mdat, - startPTS: firstPTS / timeScale, - endPTS: (lastPTS + mp4SampleDuration) / timeScale, + startPTS: minPTS / timeScale, + endPTS: (maxPTS + mp4SampleDuration) / timeScale, startDTS: firstDTS / timeScale, endDTS: this.nextAvcDts / timeScale, type: 'video', From 8708ba17c472d3430e65c8f9d180fdfca2874784 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Thu, 9 Jul 2020 22:28:07 -0400 Subject: [PATCH 2/6] Do not drop audio frames when switching qualities since the overlap detection is not based on what was previously appeneded --- src/remux/mp4-remuxer.js | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/remux/mp4-remuxer.js b/src/remux/mp4-remuxer.js index fb34ed0e9a1..41824e74e37 100644 --- a/src/remux/mp4-remuxer.js +++ b/src/remux/mp4-remuxer.js @@ -258,22 +258,21 @@ class MP4Remuxer { firstDTS = inputSamples[0].dts; lastDTS = inputSamples[inputSamples.length - 1].dts; - // check timestamp continuity accross consecutive fragments (this is to remove inter-fragment gap/hole) - let delta = firstDTS - nextAvcDts; + // check timestamp continuity across consecutive fragments (this is to remove inter-fragment gap/hole) + const delta = firstDTS - nextAvcDts; // if fragment are contiguous, detect hole/overlapping between fragments if (contiguous) { - if (delta) { - if (delta > 1) { - logger.log(`AVC: ${toMsFromMpegTsClock(delta, true)} ms hole between fragments detected,filling it`); - } else if (delta < -1) { - logger.log(`AVC: ${toMsFromMpegTsClock(-delta, true)} ms overlapping between fragments detected`); + const foundHole = delta >= 1; + const foundOverlap = delta < -1; + if (foundHole || foundOverlap) { + if (foundHole) { + logger.warn(`AVC: ${toMsFromMpegTsClock(delta, true)} ms hole between fragments detected, filling it`); + } else { + logger.warn(`AVC: ${toMsFromMpegTsClock(-delta, true)} ms overlapping between fragments detected`); } - - // remove hole/gap : set DTS to next expected DTS firstDTS = nextAvcDts; + minPTS -= delta; inputSamples[0].dts = firstDTS; - // offset PTS as well, ensure that PTS is smaller or equal than new DTS - minPTS = Math.max(minPTS - delta, nextAvcDts); inputSamples[0].pts = minPTS; logger.log(`Video: PTS/DTS adjusted: ${toMsFromMpegTsClock(minPTS, true)}/${toMsFromMpegTsClock(firstDTS, true)}, delta: ${toMsFromMpegTsClock(delta, true)} ms`); } @@ -503,9 +502,17 @@ class MP4Remuxer { // If we're overlapping by more than a duration, drop this sample if (delta <= -maxAudioFramesDrift * inputSampleDuration) { - logger.warn(`Dropping 1 audio frame @ ${toMsFromMpegTsClock(nextPts, true)} ms due to ${toMsFromMpegTsClock(delta, true)} ms overlap.`); - inputSamples.splice(i, 1); - // Don't touch nextPtsNorm or i + if (contiguous) { + logger.warn(`Dropping 1 audio frame @ ${toMsFromMpegTsClock(nextPts, true) / 1000}s due to ${toMsFromMpegTsClock(delta, true)} ms overlap.`); + inputSamples.splice(i, 1); + // Don't touch nextPtsNorm or i + } else { + // When changing qualities we can't trust that audio has been appended up to nextAudioPts + // Warn about the overlap but do not drop samples as that can introduce buffer gaps + logger.warn(`Audio frame @ ${toMsFromMpegTsClock(pts, true) / 1000}s overlaps nextAudioPts by ${toMsFromMpegTsClock(delta, true)} ms.`); + nextPts = pts + inputSampleDuration; + i++; + } } // eslint-disable-line brace-style // Insert missing frames if: @@ -514,7 +521,7 @@ class MP4Remuxer { // 3: currentTime (aka nextPtsNorm) is not 0 else if (delta >= maxAudioFramesDrift * inputSampleDuration && delta < MAX_SILENT_FRAME_DURATION_90KHZ && nextPts) { let missing = Math.round(delta / inputSampleDuration); - logger.warn(`Injecting ${missing} audio frames @ ${toMsFromMpegTsClock(nextPts, true)} ms due to ${toMsFromMpegTsClock(delta, true)} ms gap.`); + logger.warn(`Injecting ${missing} audio frames @ ${toMsFromMpegTsClock(nextPts, true) / 1000}s due to ${toMsFromMpegTsClock(delta, true)} ms gap.`); for (let j = 0; j < missing; j++) { let newStamp = Math.max(nextPts, 0); fillFrame = AAC.getSilentFrame(track.manifestCodec || track.codec, track.channelCount); From 177034f6d31253031ca1c5399150c7909c0eef3a Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Fri, 10 Jul 2020 18:57:56 -0400 Subject: [PATCH 3/6] Do not chang starting DTS when PTS < DTS is found --- src/remux/mp4-remuxer.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/remux/mp4-remuxer.js b/src/remux/mp4-remuxer.js index 41824e74e37..645f1c8ee8c 100644 --- a/src/remux/mp4-remuxer.js +++ b/src/remux/mp4-remuxer.js @@ -8,7 +8,7 @@ import MP4 from './mp4-generator'; import Event from '../events'; import { ErrorTypes, ErrorDetails } from '../errors'; -import { toMsFromMpegTsClock, toMpegTsClockFromTimescale, toTimescaleFromScale } from '../utils/timescale-conversion'; +import { toMsFromMpegTsClock, toMpegTsClockFromTimescale } from '../utils/timescale-conversion'; import { logger } from '../utils/logger'; @@ -250,7 +250,7 @@ class MP4Remuxer { if (PTSDTSshift < 0) { logger.warn(`PTS < DTS detected in video samples, shifting DTS by ${toMsFromMpegTsClock(PTSDTSshift, true)} ms to overcome this issue`); for (let i = 0; i < inputSamples.length; i++) { - inputSamples[i].dts += PTSDTSshift; + inputSamples[i].dts = Math.max(0, inputSamples[i].dts + PTSDTSshift); } } @@ -565,7 +565,7 @@ class MP4Remuxer { // logger.log(`Audio/PTS:${toMsFromMpegTsClock(pts, true)}`); // if not first sample - if (lastPTS !== undefined) { + if (lastPTS !== undefined && mp4Sample) { mp4Sample.duration = Math.round((pts - lastPTS) / scaleFactor); } else { let delta = pts - nextAudioPts; From 05ce6a7c1b4aac53f26ee3b86138929f3ac6190e Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Fri, 10 Jul 2020 22:45:13 -0400 Subject: [PATCH 4/6] Reset demuxer when backtracking --- src/controller/stream-controller.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/controller/stream-controller.js b/src/controller/stream-controller.js index b77778f61c9..2ef77a010f2 100644 --- a/src/controller/stream-controller.js +++ b/src/controller/stream-controller.js @@ -1029,6 +1029,10 @@ class StreamController extends BaseStreamController { this.nextLoadPosition = data.startPTS; this.state = State.IDLE; this.fragPrevious = frag; + if (this.demuxer) { + this.demuxer.destroy(); + this.demuxer = null; + } this.tick(); return; } From 052feab9f3a3998e95525e5aab92234d82accd6a Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Mon, 13 Jul 2020 12:14:14 -0400 Subject: [PATCH 5/6] Clamp DTS after timecode hole/overlap check and Safari frame duration calculation --- src/remux/mp4-remuxer.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/remux/mp4-remuxer.js b/src/remux/mp4-remuxer.js index 645f1c8ee8c..875473e29be 100644 --- a/src/remux/mp4-remuxer.js +++ b/src/remux/mp4-remuxer.js @@ -285,6 +285,10 @@ class MP4Remuxer { mp4SampleDuration = Math.round((lastDTS - firstDTS) / (inputSamples.length - 1)); } + // Clamp first DTS to 0 so that we're still aligning on initPTS, + // and not passing negative values to MP4.traf. This will change initial frame compositionTimeOffset! + firstDTS = Math.max(firstDTS, 0); + let nbNalu = 0, naluLen = 0; for (let i = 0; i < nbSamples; i++) { // compute total/avc sample length and nb of NAL units From b1d09643ff8fbaf042c1e8ac6fd11aadb7381030 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Mon, 13 Jul 2020 12:16:05 -0400 Subject: [PATCH 6/6] Adjust timecode hole tolerance to account for 59.94 and 29.97 framerate variance --- src/remux/mp4-remuxer.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/remux/mp4-remuxer.js b/src/remux/mp4-remuxer.js index 875473e29be..ccd2462db0e 100644 --- a/src/remux/mp4-remuxer.js +++ b/src/remux/mp4-remuxer.js @@ -262,13 +262,13 @@ class MP4Remuxer { const delta = firstDTS - nextAvcDts; // if fragment are contiguous, detect hole/overlapping between fragments if (contiguous) { - const foundHole = delta >= 1; + const foundHole = delta > 2; const foundOverlap = delta < -1; if (foundHole || foundOverlap) { if (foundHole) { - logger.warn(`AVC: ${toMsFromMpegTsClock(delta, true)} ms hole between fragments detected, filling it`); + logger.warn(`AVC: ${toMsFromMpegTsClock(delta, true)}ms (${delta}dts) hole between fragments detected, filling it`); } else { - logger.warn(`AVC: ${toMsFromMpegTsClock(-delta, true)} ms overlapping between fragments detected`); + logger.warn(`AVC: ${toMsFromMpegTsClock(-delta, true)}ms (${delta}dts) overlapping between fragments detected`); } firstDTS = nextAvcDts; minPTS -= delta;