diff --git a/packages/compiler-cli/src/ngtsc/program.ts b/packages/compiler-cli/src/ngtsc/program.ts
index 50f3cc06fd95f..55f1d4871d99b 100644
--- a/packages/compiler-cli/src/ngtsc/program.ts
+++ b/packages/compiler-cli/src/ngtsc/program.ts
@@ -441,7 +441,8 @@ export class NgtscProgram implements api.Program {
         // - error TS2531: Object is possibly 'null'.
         // - error TS2339: Property 'value' does not exist on type 'EventTarget'.
         checkTypeOfDomEvents: false,
-        checkTypeOfReferences: true,
+        checkTypeOfDomReferences: true,
+        checkTypeOfNonDomReferences: true,
         checkTypeOfPipes: true,
         strictSafeNavigationTypes: true,
       };
@@ -457,7 +458,8 @@ export class NgtscProgram implements api.Program {
         checkTypeOfOutputEvents: false,
         checkTypeOfAnimationEvents: false,
         checkTypeOfDomEvents: false,
-        checkTypeOfReferences: false,
+        checkTypeOfDomReferences: false,
+        checkTypeOfNonDomReferences: false,
         checkTypeOfPipes: false,
         strictSafeNavigationTypes: false,
       };
diff --git a/packages/compiler-cli/src/ngtsc/typecheck/src/api.ts b/packages/compiler-cli/src/ngtsc/typecheck/src/api.ts
index 77e2464792ed9..8ef1f71bc8c40 100644
--- a/packages/compiler-cli/src/ngtsc/typecheck/src/api.ts
+++ b/packages/compiler-cli/src/ngtsc/typecheck/src/api.ts
@@ -154,14 +154,24 @@ export interface TypeCheckingConfig {
    */
   checkTypeOfDomEvents: boolean;
 
+  /**
+   * Whether to infer the type of local references to DOM elements.
+   *
+   * If this is `true`, the type of a `#ref` variable on a DOM node in the template will be
+   * determined by the type of `document.createElement` for the given DOM node type. If set to
+   * `false`, the type of `ref` for DOM nodes will be `any`.
+   */
+  checkTypeOfDomReferences: boolean;
+
+
   /**
    * Whether to infer the type of local references.
    *
-   * If this is `true`, the type of any `#ref` variable in the template will be determined by the
-   * referenced entity (either a directive or a DOM element). If set to `false`, the type of `ref`
-   * will be `any`.
+   * If this is `true`, the type of a `#ref` variable that points to a directive or `TemplateRef` in
+   * the template will be inferred correctly. If set to `false`, the type of `ref` for will be
+   * `any`.
    */
-  checkTypeOfReferences: boolean;
+  checkTypeOfNonDomReferences: boolean;
 
   /**
    * Whether to include type information from pipes in the type-checking operation.
diff --git a/packages/compiler-cli/src/ngtsc/typecheck/src/type_check_block.ts b/packages/compiler-cli/src/ngtsc/typecheck/src/type_check_block.ts
index e28e93d2c00db..384dfa1d8ff47 100644
--- a/packages/compiler-cli/src/ngtsc/typecheck/src/type_check_block.ts
+++ b/packages/compiler-cli/src/ngtsc/typecheck/src/type_check_block.ts
@@ -1023,11 +1023,6 @@ class TcbExpressionTranslator {
       addParseSpanInfo(expr, toAbsoluteSpan(ast.span, this.sourceSpan));
       return expr;
     } else if (binding instanceof TmplAstReference) {
-      if (!this.tcb.env.config.checkTypeOfReferences) {
-        // References are pinned to 'any'.
-        return NULL_AS_ANY;
-      }
-
       const target = this.tcb.boundTarget.getReferenceTarget(binding);
       if (target === null) {
         throw new Error(`Unbound reference? ${binding.name}`);
@@ -1037,10 +1032,20 @@ class TcbExpressionTranslator {
       // element or template.
 
       if (target instanceof TmplAstElement) {
+        if (!this.tcb.env.config.checkTypeOfDomReferences) {
+          // References to DOM nodes are pinned to 'any'.
+          return NULL_AS_ANY;
+        }
+
         const expr = ts.getMutableClone(this.scope.resolve(target));
         addParseSpanInfo(expr, toAbsoluteSpan(ast.span, this.sourceSpan));
         return expr;
       } else if (target instanceof TmplAstTemplate) {
+        if (!this.tcb.env.config.checkTypeOfNonDomReferences) {
+          // References to `TemplateRef`s pinned to 'any'.
+          return NULL_AS_ANY;
+        }
+
         // Direct references to an <ng-template> node simply require a value of type
         // `TemplateRef<any>`. To get this, an expression of the form
         // `(null as any as TemplateRef<any>)` is constructed.
@@ -1053,6 +1058,11 @@ class TcbExpressionTranslator {
         addParseSpanInfo(value, toAbsoluteSpan(ast.span, this.sourceSpan));
         return value;
       } else {
+        if (!this.tcb.env.config.checkTypeOfNonDomReferences) {
+          // References to directives are pinned to 'any'.
+          return NULL_AS_ANY;
+        }
+
         const expr = ts.getMutableClone(this.scope.resolve(target.node, target.directive));
         addParseSpanInfo(expr, toAbsoluteSpan(ast.span, this.sourceSpan));
         return expr;
diff --git a/packages/compiler-cli/src/ngtsc/typecheck/test/test_utils.ts b/packages/compiler-cli/src/ngtsc/typecheck/test/test_utils.ts
index 7d9eb95728960..3945b02d1d6fb 100644
--- a/packages/compiler-cli/src/ngtsc/typecheck/test/test_utils.ts
+++ b/packages/compiler-cli/src/ngtsc/typecheck/test/test_utils.ts
@@ -156,7 +156,8 @@ export const ALL_ENABLED_CONFIG: TypeCheckingConfig = {
   checkTypeOfOutputEvents: true,
   checkTypeOfAnimationEvents: true,
   checkTypeOfDomEvents: true,
-  checkTypeOfReferences: true,
+  checkTypeOfDomReferences: true,
+  checkTypeOfNonDomReferences: true,
   checkTypeOfPipes: true,
   strictSafeNavigationTypes: true,
 };
@@ -207,7 +208,8 @@ export function tcb(
     checkTypeOfOutputEvents: true,
     checkTypeOfAnimationEvents: true,
     checkTypeOfDomEvents: true,
-    checkTypeOfReferences: true,
+    checkTypeOfDomReferences: true,
+    checkTypeOfNonDomReferences: true,
     checkTypeOfPipes: true,
     checkTemplateBodies: true,
     strictSafeNavigationTypes: true,
diff --git a/packages/compiler-cli/src/ngtsc/typecheck/test/type_check_block_spec.ts b/packages/compiler-cli/src/ngtsc/typecheck/test/type_check_block_spec.ts
index 5e13b1b145282..9343450425311 100644
--- a/packages/compiler-cli/src/ngtsc/typecheck/test/type_check_block_spec.ts
+++ b/packages/compiler-cli/src/ngtsc/typecheck/test/type_check_block_spec.ts
@@ -298,7 +298,8 @@ describe('type check blocks', () => {
       checkTypeOfOutputEvents: true,
       checkTypeOfAnimationEvents: true,
       checkTypeOfDomEvents: true,
-      checkTypeOfReferences: true,
+      checkTypeOfDomReferences: true,
+      checkTypeOfNonDomReferences: true,
       checkTypeOfPipes: true,
       strictSafeNavigationTypes: true,
     };
@@ -424,7 +425,7 @@ describe('type check blocks', () => {
       });
     });
 
-    describe('config.checkTypeOfReferences', () => {
+    describe('config.checkTypeOfDomReferences', () => {
       const TEMPLATE = `<input #ref>{{ref.value}}`;
 
       it('should trace references when enabled', () => {
@@ -433,12 +434,44 @@ describe('type check blocks', () => {
       });
 
       it('should use any for reference types when disabled', () => {
-        const DISABLED_CONFIG: TypeCheckingConfig = {...BASE_CONFIG, checkTypeOfReferences: false};
+        const DISABLED_CONFIG:
+            TypeCheckingConfig = {...BASE_CONFIG, checkTypeOfDomReferences: false};
         const block = tcb(TEMPLATE, [], DISABLED_CONFIG);
         expect(block).toContain('(null as any).value');
       });
     });
 
+    describe('config.checkTypeOfNonDomReferences', () => {
+      const DIRECTIVES: TestDeclaration[] = [{
+        type: 'directive',
+        name: 'Dir',
+        selector: '[dir]',
+        exportAs: ['dir'],
+        inputs: {'dirInput': 'dirInput'},
+        outputs: {'outputField': 'dirOutput'},
+        hasNgTemplateContextGuard: true,
+      }];
+      const TEMPLATE =
+          `<div dir #ref="dir">{{ref.value}}</div><ng-template #ref2></ng-template>{{ref2.value2}}`;
+
+      it('should trace references to a directive when enabled', () => {
+        const block = tcb(TEMPLATE, DIRECTIVES);
+        expect(block).toContain('(_t2).value');
+      });
+
+      it('should trace references to an <ng-template> when enabled', () => {
+        const block = tcb(TEMPLATE, DIRECTIVES);
+        expect(block).toContain('((null as any as core.TemplateRef<any>)).value2');
+      });
+
+      it('should use any for reference types when disabled', () => {
+        const DISABLED_CONFIG:
+            TypeCheckingConfig = {...BASE_CONFIG, checkTypeOfNonDomReferences: false};
+        const block = tcb(TEMPLATE, DIRECTIVES, DISABLED_CONFIG);
+        expect(block).toContain('(null as any).value');
+      });
+    });
+
     describe('config.checkTypeOfAttributes', () => {
       const TEMPLATE = `<textarea dir disabled cols="3" [rows]="2">{{ref.value}}</textarea>`;
       const DIRECTIVES: TestDeclaration[] = [{