diff --git a/src/controller/audio-stream-controller.js b/src/controller/audio-stream-controller.js
index b12e53cc0ae..9959dd1a9cc 100644
--- a/src/controller/audio-stream-controller.js
+++ b/src/controller/audio-stream-controller.js
@@ -381,6 +381,7 @@ class AudioStreamController extends BaseStreamController {
     }
     this.media = this.mediaBuffer = this.videoBuffer = null;
     this.loadedmetadata = false;
+    this.fragmentTracker.removeAllFragments();
     this.stopLoad();
   }
 
diff --git a/src/controller/buffer-controller.ts b/src/controller/buffer-controller.ts
index 89287541893..62d58ea1649 100644
--- a/src/controller/buffer-controller.ts
+++ b/src/controller/buffer-controller.ts
@@ -55,6 +55,9 @@ class BufferController extends EventHandler {
   // The number of BUFFER_CODEC events received before any sourceBuffers are created
   public bufferCodecEventsExpected: number = 0;
 
+  // The total number of BUFFER_CODEC events received
+  private _bufferCodecEventsTotal: number = 0;
+
   // A reference to the attached media element
   public media: HTMLMediaElement | null = null;
 
@@ -143,7 +146,7 @@ class BufferController extends EventHandler {
     // sourcebuffers will be created all at once when the expected nb of tracks will be reached
     // in case alt audio is not used, only one BUFFER_CODEC event will be fired from main stream controller
     // it will contain the expected nb of source buffers, no need to compute it
-    this.bufferCodecEventsExpected = data.altAudio ? 2 : 1;
+    this.bufferCodecEventsExpected = this._bufferCodecEventsTotal = data.altAudio ? 2 : 1;
     logger.log(`${this.bufferCodecEventsExpected} bufferCodec event(s) expected`);
   }
 
@@ -202,6 +205,7 @@ class BufferController extends EventHandler {
       this.mediaSource = null;
       this.media = null;
       this._objectUrl = null;
+      this.bufferCodecEventsExpected = this._bufferCodecEventsTotal;
       this.pendingTracks = {};
       this.tracks = {};
       this.sourceBuffer = {};
diff --git a/src/controller/stream-controller.js b/src/controller/stream-controller.js
index 092a55bcc77..2c663f9dcba 100644
--- a/src/controller/stream-controller.js
+++ b/src/controller/stream-controller.js
@@ -47,6 +47,7 @@ class StreamController extends BaseStreamController {
     this._state = State.STOPPED;
     this.stallReported = false;
     this.gapController = null;
+    this.altAudio = false;
   }
 
   startLoad (startPosition) {
@@ -691,6 +692,7 @@ class StreamController extends BaseStreamController {
       media.removeEventListener('ended', this.onvended);
       this.onvseeking = this.onvseeked = this.onvended = null;
     }
+    this.fragmentTracker.removeAllFragments();
     this.media = this.mediaBuffer = null;
     this.loadedmetadata = false;
     this.stopLoad();
@@ -735,6 +737,7 @@ class StreamController extends BaseStreamController {
       logger.log('both AAC/HE-AAC audio found in levels; declaring level codec as HE-AAC');
     }
 
+    this.altAudio = data.altAudio;
     this.levels = data.levels;
     this.startFragRequested = false;
     let config = this.config;
diff --git a/src/controller/subtitle-stream-controller.js b/src/controller/subtitle-stream-controller.js
index ac8e0b872ab..8e313d9d8b9 100644
--- a/src/controller/subtitle-stream-controller.js
+++ b/src/controller/subtitle-stream-controller.js
@@ -84,6 +84,11 @@ export class SubtitleStreamController extends BaseStreamController {
 
   onMediaDetaching () {
     this.media.removeEventListener('seeking', this._onMediaSeeking);
+    this.fragmentTracker.removeAllFragments();
+    this.currentTrackId = -1;
+    this.tracks.forEach((track) => {
+      this.tracksBuffered[track.id] = [];
+    });
     this.media = null;
     this.state = State.STOPPED;
   }
@@ -111,7 +116,7 @@ export class SubtitleStreamController extends BaseStreamController {
   onSubtitleTrackSwitch (data) {
     this.currentTrackId = data.id;
 
-    if (!this.tracks || this.currentTrackId === -1) {
+    if (!this.tracks || !this.tracks.length || this.currentTrackId === -1) {
       this.clearInterval();
       return;
     }
diff --git a/src/controller/subtitle-track-controller.js b/src/controller/subtitle-track-controller.js
index 9da39317b04..717c16e0e21 100644
--- a/src/controller/subtitle-track-controller.js
+++ b/src/controller/subtitle-track-controller.js
@@ -66,6 +66,11 @@ class SubtitleTrackController extends EventHandler {
       this.media.textTracks.removeEventListener('change', this.trackChangeListener);
     }
 
+    if (Number.isFinite(this.subtitleTrack)) {
+      this.queuedDefaultTrack = this.subtitleTrack;
+    }
+
+    this.trackId = -1;
     this.media = null;
   }
 
diff --git a/tests/unit/controller/subtitle-stream-controller.js b/tests/unit/controller/subtitle-stream-controller.js
index 1dc281b7fe4..37c33a13887 100644
--- a/tests/unit/controller/subtitle-stream-controller.js
+++ b/tests/unit/controller/subtitle-stream-controller.js
@@ -61,7 +61,7 @@ describe('SubtitleStreamController', function () {
     });
 
     it('should call clearInterval if no tracks present', function () {
-      subtitleStreamController.tracks = null;
+      subtitleStreamController.tracks = [];
       hls.trigger(Event.SUBTITLE_TRACK_SWITCH, {
         id: 0
       });