diff --git a/src/compiler/tsbuild.ts b/src/compiler/tsbuild.ts
index aed4ba48e94b0..a5eae8922da30 100644
--- a/src/compiler/tsbuild.ts
+++ b/src/compiler/tsbuild.ts
@@ -403,10 +403,20 @@ namespace ts {
 
         compilerHost.resolveModuleNames = maybeBind(host, host.resolveModuleNames);
         compilerHost.resolveTypeReferenceDirectives = maybeBind(host, host.resolveTypeReferenceDirectives);
-        let moduleResolutionCache = !compilerHost.resolveModuleNames ? createModuleResolutionCache(currentDirectory, getCanonicalFileName) : undefined;
+        const moduleResolutionCache = !compilerHost.resolveModuleNames ? createModuleResolutionCache(currentDirectory, getCanonicalFileName) : undefined;
+        let cacheState: {
+            originalReadFile: CompilerHost["readFile"];
+            originalFileExists: CompilerHost["fileExists"];
+            originalDirectoryExists: CompilerHost["directoryExists"];
+            originalCreateDirectory: CompilerHost["createDirectory"];
+            originalWriteFile: CompilerHost["writeFile"] | undefined;
+            originalReadFileWithCache: CompilerHost["readFile"];
+            originalGetSourceFile: CompilerHost["getSourceFile"];
+            originalResolveModuleNames: CompilerHost["resolveModuleNames"];
+        } | undefined;
 
         const buildInfoChecked = createFileMap<true>(toPath);
-        let extendedConfigCache: Map<ExtendedConfigCacheEntry> | undefined;
+        const extendedConfigCache = createMap<ExtendedConfigCacheEntry>();
 
         // Watch state
         const builderPrograms = createFileMap<T>(toPath);
@@ -869,6 +879,7 @@ namespace ts {
             diagnostics.removeKey(resolved);
 
             addProjToQueue(resolved, reloadLevel);
+            enableCache();
         }
 
         /**
@@ -929,6 +940,7 @@ namespace ts {
                     }
                 }
                 else {
+                    disableCache();
                     reportErrorSummary();
                 }
             }
@@ -1384,20 +1396,20 @@ namespace ts {
             return configFileNames.map(resolveProjectName);
         }
 
-        function buildAllProjects(): ExitStatus {
-            if (options.watch) { reportWatchStatus(Diagnostics.Starting_compilation_in_watch_mode); }
-            // TODO:: In watch mode as well to use caches for incremental build once we can invalidate caches correctly and have right api
-            // Override readFile for json files and output .d.ts to cache the text
-            const savedReadFileWithCache = readFileWithCache;
-            const savedGetSourceFile = compilerHost.getSourceFile;
+        function enableCache() {
+            if (cacheState) {
+                disableCache();
+            }
+
+            const originalReadFileWithCache = readFileWithCache;
+            const originalGetSourceFile = compilerHost.getSourceFile;
 
             const { originalReadFile, originalFileExists, originalDirectoryExists,
                 originalCreateDirectory, originalWriteFile, getSourceFileWithCache,
                 readFileWithCache: newReadFileWithCache
-            } = changeCompilerHostLikeToUseCache(host, toPath, (...args) => savedGetSourceFile.call(compilerHost, ...args));
+            } = changeCompilerHostLikeToUseCache(host, toPath, (...args) => originalGetSourceFile.call(compilerHost, ...args));
             readFileWithCache = newReadFileWithCache;
             compilerHost.getSourceFile = getSourceFileWithCache!;
-            extendedConfigCache = createMap();
 
             const originalResolveModuleNames = compilerHost.resolveModuleNames;
             if (!compilerHost.resolveModuleNames) {
@@ -1406,6 +1418,41 @@ namespace ts {
                     loadWithLocalCache<ResolvedModuleFull>(Debug.assertEachDefined(moduleNames), containingFile, redirectedReference, loader);
             }
 
+            cacheState = {
+                originalReadFile,
+                originalFileExists,
+                originalDirectoryExists,
+                originalCreateDirectory,
+                originalWriteFile,
+                originalReadFileWithCache,
+                originalGetSourceFile,
+                originalResolveModuleNames
+            };
+        }
+
+        function disableCache() {
+            if (!cacheState) return;
+
+            host.readFile = cacheState.originalReadFile;
+            host.fileExists = cacheState.originalFileExists;
+            host.directoryExists = cacheState.originalDirectoryExists;
+            host.createDirectory = cacheState.originalCreateDirectory;
+            host.writeFile = cacheState.originalWriteFile;
+            compilerHost.getSourceFile = cacheState.originalGetSourceFile;
+            readFileWithCache = cacheState.originalReadFileWithCache;
+            compilerHost.resolveModuleNames = cacheState.originalResolveModuleNames;
+            extendedConfigCache.clear();
+            if (moduleResolutionCache) {
+                moduleResolutionCache.directoryToModuleNameMap.clear();
+                moduleResolutionCache.moduleNameToDirectoryMap.clear();
+            }
+            cacheState = undefined;
+        }
+
+        function buildAllProjects(): ExitStatus {
+            if (options.watch) { reportWatchStatus(Diagnostics.Starting_compilation_in_watch_mode); }
+            enableCache();
+
             const graph = getGlobalDependencyGraph();
             reportBuildQueue(graph);
             let anyFailed = false;
@@ -1459,16 +1506,7 @@ namespace ts {
                 anyFailed = anyFailed || !!(buildResult & BuildResultFlags.AnyErrors);
             }
             reportErrorSummary();
-            host.readFile = originalReadFile;
-            host.fileExists = originalFileExists;
-            host.directoryExists = originalDirectoryExists;
-            host.createDirectory = originalCreateDirectory;
-            host.writeFile = originalWriteFile;
-            compilerHost.getSourceFile = savedGetSourceFile;
-            readFileWithCache = savedReadFileWithCache;
-            extendedConfigCache = undefined;
-            compilerHost.resolveModuleNames = originalResolveModuleNames;
-            moduleResolutionCache = undefined;
+            disableCache();
             return anyFailed ? ExitStatus.DiagnosticsPresent_OutputsSkipped : ExitStatus.Success;
         }