diff --git a/src/remux/mp4-remuxer.ts b/src/remux/mp4-remuxer.ts
index b35a248c2dd..d4cdd130d42 100644
--- a/src/remux/mp4-remuxer.ts
+++ b/src/remux/mp4-remuxer.ts
@@ -275,6 +275,8 @@ export default class MP4Remuxer implements Remuxer {
     // PTSNormalize will make PTS/DTS value monotonic, we use last known DTS value as reference value
     for (let i = 0; i < nbSamples; i++) {
       const sample = inputSamples[i];
+      sample.pts = PTSNormalize(sample.pts - initPTS, nextAvcDts);
+      sample.dts = PTSNormalize(sample.dts - initPTS, nextAvcDts);
       if (sample.dts > sample.pts) {
         const PTS_DTS_SHIFT_TOLERANCE_90KHZ = 90000 * 0.2;
         ptsDtsShift = Math.max(Math.min(ptsDtsShift, sample.pts - sample.dts), -1 * PTS_DTS_SHIFT_TOLERANCE_90KHZ);
@@ -282,8 +284,6 @@ export default class MP4Remuxer implements Remuxer {
       if (sample.dts < inputSamples[i > 0 ? i - 1 : i].dts) {
         sortSamples = true;
       }
-      sample.pts = PTSNormalize(sample.pts - initPTS, nextAvcDts);
-      sample.dts = PTSNormalize(sample.dts - initPTS, nextAvcDts);
     }
 
     // sort video samples by DTS then PTS then demux id order