diff --git a/src/index.js b/src/index.js
index 49d458ff..60e909d2 100644
--- a/src/index.js
+++ b/src/index.js
@@ -359,6 +359,8 @@ class MiniCssExtractPlugin {
                     'var request = event && event.target && event.target.src || fullhref;',
                     'var err = new Error("Loading CSS chunk " + chunkId + " failed.\\n(" + request + ")");',
                     'err.request = request;',
+                    'delete installedCssChunks[chunkId]',
+                    'linkTag.parentNode.removeChild(linkTag)',
                     'reject(err);',
                   ]),
                   '};',
diff --git a/test/manual/index.html b/test/manual/index.html
index e49d208a..8b29a314 100644
--- a/test/manual/index.html
+++ b/test/manual/index.html
@@ -34,6 +34,11 @@
     <p>But turn orange, when <button class="lazy-button2">pressing this button</button>. Additional clicks have no effect.</p>
     <p>Refresh and press buttons in reverse order: This should turn green instead.</p>
   </div>
+  <div class="test lazy-failure-css">
+    <p>Lazy CSS: Turn off the network and <button class="lazy-failure-button">press this button</button>.</p>
+    <p>An error should have appeared.</p>
+    <p>Now if you turn the network back on and click it again, it should turn aqua.</p>
+  </div>
   <div class="test preloaded-css1">
     <p>Preloaded CSS: Must be green.</p>
     <p><button class="preloaded-button1">Pressing this button</button> displays an alert and should turn red.</p>
diff --git a/test/manual/src/index.js b/test/manual/src/index.js
index 3f4bd150..c6b9b72f 100644
--- a/test/manual/src/index.js
+++ b/test/manual/src/index.js
@@ -5,10 +5,12 @@ const handleError = (err) => {
   console.error(err);
 }
 
-const makeButton = (className, fn) => {
+const makeButton = (className, fn, shouldDisable = true) => {
   const button = document.querySelector(className);
   button.addEventListener("click", () => {
-    button.disabled = true;
+    if(shouldDisable) {
+      button.disabled = true;
+    }
     fn().then(() => {
       button.disabled = false;
     }).catch(handleError);
@@ -20,3 +22,5 @@ makeButton(".lazy-button2", () => import('./lazy2.css'));
 
 makeButton(".preloaded-button1", () => import(/* webpackChunkName: "preloaded1" */ './preloaded1'));
 makeButton(".preloaded-button2", () => import(/* webpackChunkName: "preloaded2" */ './preloaded2'));
+
+makeButton(".lazy-failure-button", () => import('./lazy-failure.js'), false);
diff --git a/test/manual/src/lazy-failure.css b/test/manual/src/lazy-failure.css
new file mode 100644
index 00000000..479edbac
--- /dev/null
+++ b/test/manual/src/lazy-failure.css
@@ -0,0 +1,3 @@
+.lazy-failure-css {
+  background: aqua;
+}
diff --git a/test/manual/src/lazy-failure.js b/test/manual/src/lazy-failure.js
new file mode 100644
index 00000000..cf0f24b4
--- /dev/null
+++ b/test/manual/src/lazy-failure.js
@@ -0,0 +1 @@
+import './lazy-failure.css';