diff --git a/packages/angular_devkit/build_angular/BUILD.bazel b/packages/angular_devkit/build_angular/BUILD.bazel
index d8a4228fecb2..9d7c95930a6f 100644
--- a/packages/angular_devkit/build_angular/BUILD.bazel
+++ b/packages/angular_devkit/build_angular/BUILD.bazel
@@ -94,7 +94,6 @@ ts_library(
     module_root = "src/index.d.ts",
     deps = [
         "//packages/angular_devkit/architect",
-        "//packages/angular_devkit/build_optimizer",
         "//packages/angular_devkit/build_webpack",
         "//packages/angular_devkit/core",
         "//packages/angular_devkit/core/node",
diff --git a/packages/angular_devkit/build_angular/package.json b/packages/angular_devkit/build_angular/package.json
index 916e16888cb6..95cc30f40ee4 100644
--- a/packages/angular_devkit/build_angular/package.json
+++ b/packages/angular_devkit/build_angular/package.json
@@ -8,7 +8,6 @@
   "dependencies": {
     "@ampproject/remapping": "1.0.1",
     "@angular-devkit/architect": "0.0.0",
-    "@angular-devkit/build-optimizer": "0.0.0",
     "@angular-devkit/build-webpack": "0.0.0",
     "@angular-devkit/core": "0.0.0",
     "@babel/core": "7.15.0",
diff --git a/packages/angular_devkit/build_angular/src/babel/webpack-loader.ts b/packages/angular_devkit/build_angular/src/babel/webpack-loader.ts
index ba6b2583e1c3..52d2f1444669 100644
--- a/packages/angular_devkit/build_angular/src/babel/webpack-loader.ts
+++ b/packages/angular_devkit/build_angular/src/babel/webpack-loader.ts
@@ -66,8 +66,7 @@ export default custom<AngularCustomOptions>(() => {
       const esTarget = scriptTarget as ScriptTarget | undefined;
       if (esTarget !== undefined) {
         if (esTarget < ScriptTarget.ES2015) {
-          // TypeScript files will have already been downlevelled
-          customOptions.forceES5 = !/\.tsx?$/.test(this.resourcePath);
+          customOptions.forceES5 = true;
         } else if (esTarget >= ScriptTarget.ES2017) {
           customOptions.forceAsyncTransformation =
             !/[\\\/][_f]?esm2015[\\\/]/.test(this.resourcePath) && source.includes('async');
diff --git a/packages/angular_devkit/build_angular/src/webpack/configs/common.ts b/packages/angular_devkit/build_angular/src/webpack/configs/common.ts
index f6e06d7e9750..38125ba9a14e 100644
--- a/packages/angular_devkit/build_angular/src/webpack/configs/common.ts
+++ b/packages/angular_devkit/build_angular/src/webpack/configs/common.ts
@@ -6,10 +6,6 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {
-  BuildOptimizerWebpackPlugin,
-  buildOptimizerLoaderPath,
-} from '@angular-devkit/build-optimizer';
 import {
   GLOBAL_DEFS_FOR_TERSER,
   GLOBAL_DEFS_FOR_TERSER_WITH_AOT,
@@ -301,17 +297,6 @@ export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
     });
   }
 
-  let buildOptimizerUseRule: RuleSetRule[] = [];
-  if (buildOptions.buildOptimizer && wco.scriptTarget < ScriptTarget.ES2015) {
-    extraPlugins.push(new BuildOptimizerWebpackPlugin());
-    buildOptimizerUseRule = [
-      {
-        loader: buildOptimizerLoaderPath,
-        options: { sourceMap: scriptsSourceMap },
-      },
-    ];
-  }
-
   const extraMinimizers = [];
 
   if (scriptsOptimization) {
@@ -422,10 +407,9 @@ export function getCommonConfig(wco: WebpackConfigOptions): Configuration {
                 cacheDirectory: findCachePath('babel-webpack'),
                 scriptTarget: wco.scriptTarget,
                 aot: buildOptions.aot,
-                optimize: buildOptions.buildOptimizer && wco.scriptTarget >= ScriptTarget.ES2015,
+                optimize: buildOptions.buildOptimizer,
               },
             },
-            ...buildOptimizerUseRule,
           ],
         },
         ...extraRules,
diff --git a/packages/angular_devkit/build_angular/src/webpack/configs/typescript.ts b/packages/angular_devkit/build_angular/src/webpack/configs/typescript.ts
index 4f3799321c94..3afff245b9e6 100644
--- a/packages/angular_devkit/build_angular/src/webpack/configs/typescript.ts
+++ b/packages/angular_devkit/build_angular/src/webpack/configs/typescript.ts
@@ -9,6 +9,7 @@
 import { getSystemPath } from '@angular-devkit/core';
 import { CompilerOptions } from '@angular/compiler-cli';
 import { AngularWebpackLoaderPath, AngularWebpackPlugin } from '@ngtools/webpack';
+import { ScriptTarget } from 'typescript';
 import { Configuration } from 'webpack';
 import { WebpackConfigOptions } from '../../utils/build-options';
 
@@ -45,6 +46,13 @@ function createIvyPlugin(
     compilerOptions.preserveSymlinks = buildOptions.preserveSymlinks;
   }
 
+  // Outputting ES2015 from TypeScript is the required minimum for the build optimizer passes.
+  // Downleveling to ES5 will occur after the build optimizer passes via babel which is the same
+  // as for third-party libraries. This greatly reduces the complexity of static analysis.
+  if (wco.scriptTarget < ScriptTarget.ES2015) {
+    compilerOptions.target = ScriptTarget.ES2015;
+  }
+
   const fileReplacements: Record<string, string> = {};
   if (buildOptions.fileReplacements) {
     for (const replacement of buildOptions.fileReplacements) {