diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts
index 3d6a46c6920..9cc7f920aa8 100644
--- a/src/controller/stream-controller.ts
+++ b/src/controller/stream-controller.ts
@@ -935,7 +935,10 @@ export default class StreamController
     event: Events.BUFFER_FLUSHED,
     { type }: BufferFlushedData
   ) {
-    if (type !== ElementaryStreamTypes.AUDIO) {
+    if (
+      type !== ElementaryStreamTypes.AUDIO ||
+      (this.audioOnly && !this.altAudio)
+    ) {
       const media =
         (type === ElementaryStreamTypes.VIDEO
           ? this.videoBuffer