Reset loading start on worker setup error

Fixes #5617

(cherry picked from commit 03bafc6)

src/controller/base-stream-controller.ts

@@ -1503,10 +1503,7 @@ export default class BaseStreamController
       action === NetworkErrorAction.RetryRequest &&
       retryConfig
     ) {
-      if (!this.loadedmetadata) {
-        this.startFragRequested = false;
-        this.nextLoadPosition = this.startPosition;
-      }
+      this.resetStartWhenNotLoaded(this.levelLastLoaded ?? frag.level);
       const delay = getRetryDelay(retryConfig, retryCount);
       this.warn(
         `Fragment ${frag.sn} of ${filterType} ${frag.level} errored with ${
@@ -1639,7 +1636,7 @@ export default class BaseStreamController
       `The loading context changed while buffering fragment ${chunkMeta.sn} of level ${chunkMeta.level}. This chunk will not be buffered.`
     );
     this.removeUnbufferedFrags();
-    this.resetStartWhenNotLoaded(chunkMeta.level);
+    this.resetStartWhenNotLoaded(this.levelLastLoaded ?? chunkMeta.level);
     this.resetLoadingState();
   }
 
@@ -1737,7 +1734,11 @@ export default class BaseStreamController
 
   protected recoverWorkerError(data: ErrorData) {
     if (data.event === 'demuxerWorker') {
+      this.fragmentTracker.removeAllFragments();
       this.resetTransmuxer();
+      this.resetStartWhenNotLoaded(
+        this.levelLastLoaded ?? this.fragCurrent?.level ?? 0
+      );
       this.resetLoadingState();
     }
   }