diff --git a/packages/@ember/-internals/glimmer/index.ts b/packages/@ember/-internals/glimmer/index.ts
index beb9b2f0b23..0a65246c8a2 100644
--- a/packages/@ember/-internals/glimmer/index.ts
+++ b/packages/@ember/-internals/glimmer/index.ts
@@ -404,8 +404,12 @@ export { INVOKE } from './lib/helpers/action';
 export { default as OutletView } from './lib/views/outlet';
 export { OutletState } from './lib/utils/outlet';
 export { capabilities } from './lib/component-managers/custom';
-export { setComponentManager, getComponentManager } from './lib/utils/custom-component-manager';
-export { setModifierManager, getModifierManager } from './lib/utils/custom-modifier-manager';
+export {
+  setComponentManager,
+  getComponentManager,
+  setModifierManager,
+  getModifierManager,
+} from './lib/utils/managers';
 export { capabilities as modifierCapabilities } from './lib/modifiers/custom';
 export { isSerializationFirstNode } from './lib/utils/serialization-first-node-helpers';
 export { setComponentTemplate, getComponentTemplate } from './lib/utils/component-template';
diff --git a/packages/@ember/-internals/glimmer/lib/component-managers/internal.ts b/packages/@ember/-internals/glimmer/lib/component-managers/internal.ts
index 42a0509a325..67b4dd15812 100644
--- a/packages/@ember/-internals/glimmer/lib/component-managers/internal.ts
+++ b/packages/@ember/-internals/glimmer/lib/component-managers/internal.ts
@@ -12,7 +12,7 @@ import {
 } from '@glimmer/interfaces';
 import { createConstRef, isConstRef, Reference, valueForRef } from '@glimmer/reference';
 import { registerDestructor } from '@glimmer/runtime';
-import { unwrapTemplate } from '@glimmer/util';
+import { _WeakSet, unwrapTemplate } from '@glimmer/util';
 import InternalComponent from '../components/internal';
 import { EmberVMEnvironment } from '../environment';
 import RuntimeResolver from '../resolver';
@@ -57,6 +57,12 @@ export class InternalComponentDefinition
   }
 }
 
+const INTERNAL_MANAGERS = new _WeakSet();
+
+export function isInternalManager(manager: object): manager is InternalManager {
+  return INTERNAL_MANAGERS.has(manager);
+}
+
 export default class InternalManager
   extends AbstractComponentManager<InternalComponentState, InternalDefinitionState>
   implements WithStaticLayout<InternalComponentState, InternalDefinitionState, RuntimeResolver> {
@@ -66,6 +72,7 @@ export default class InternalManager
 
   constructor(private owner: Owner, private name: string) {
     super();
+    INTERNAL_MANAGERS.add(this);
   }
 
   getCapabilities(): ComponentCapabilities {
diff --git a/packages/@ember/-internals/glimmer/lib/components/input.ts b/packages/@ember/-internals/glimmer/lib/components/input.ts
index fe66a32679c..bb0a64628a7 100644
--- a/packages/@ember/-internals/glimmer/lib/components/input.ts
+++ b/packages/@ember/-internals/glimmer/lib/components/input.ts
@@ -2,7 +2,7 @@
 @module @ember/component
 */
 import InternalManager from '../component-managers/internal';
-import { setManager } from '../utils/managers';
+import { setComponentManager } from '../utils/managers';
 import InternalComponent from './internal';
 
 /**
@@ -117,13 +117,6 @@ export default class Input extends InternalComponent {
   }
 }
 
-setManager(
-  {
-    factory: InternalManager.for('input'),
-    internal: true,
-    type: 'component',
-  },
-  Input
-);
+setComponentManager(InternalManager.for('input'), Input);
 
 Input.toString = () => '@ember/component/input';
diff --git a/packages/@ember/-internals/glimmer/lib/resolver.ts b/packages/@ember/-internals/glimmer/lib/resolver.ts
index a2f34872567..7ae05061e22 100644
--- a/packages/@ember/-internals/glimmer/lib/resolver.ts
+++ b/packages/@ember/-internals/glimmer/lib/resolver.ts
@@ -19,10 +19,8 @@ import {
 import { PartialDefinitionImpl } from '@glimmer/opcode-compiler';
 import { getDynamicVar, ModifierDefinition, registerDestructor } from '@glimmer/runtime';
 import { CurlyComponentDefinition } from './component-managers/curly';
-import { CustomManagerDefinition, ManagerDelegate } from './component-managers/custom';
-import InternalComponentManager, {
-  InternalComponentDefinition,
-} from './component-managers/internal';
+import { CustomManagerDefinition } from './component-managers/custom';
+import { InternalComponentDefinition, isInternalManager } from './component-managers/internal';
 import { TemplateOnlyComponentDefinition } from './component-managers/template-only';
 import InternalComponent from './components/internal';
 import { isClassHelper, isHelperFactory } from './helper';
@@ -44,14 +42,13 @@ import { default as queryParams } from './helpers/query-param';
 import { default as readonly } from './helpers/readonly';
 import { default as unbound } from './helpers/unbound';
 import ActionModifierManager from './modifiers/action';
-import { CustomModifierDefinition, ModifierManagerDelegate } from './modifiers/custom';
+import { CustomModifierDefinition } from './modifiers/custom';
 import OnModifierManager from './modifiers/on';
 import { mountHelper } from './syntax/mount';
 import { outletHelper } from './syntax/outlet';
 import { Factory as TemplateFactory, OwnedTemplate } from './template';
 import { getComponentTemplate } from './utils/component-template';
-import { getModifierManager } from './utils/custom-modifier-manager';
-import { getManager } from './utils/managers';
+import { getComponentManager, getModifierManager } from './utils/managers';
 import { createHelperRef } from './utils/references';
 
 function instrumentationPayload(name: string) {
@@ -404,10 +401,9 @@ export default class RuntimeResolverImpl implements RuntimeResolver<OwnedTemplat
       let owner = meta.owner;
       let modifier = owner.factoryFor<unknown, FactoryClass>(`modifier:${name}`);
       if (modifier !== undefined) {
-        let managerFactory = getModifierManager<ModifierManagerDelegate<unknown>>(modifier.class);
-        let manager = managerFactory!(owner);
+        let manager = getModifierManager(owner, modifier.class!);
 
-        return new CustomModifierDefinition(name, modifier, manager, this.isInteractive);
+        return new CustomModifierDefinition(name, modifier, manager!, this.isInteractive);
       }
     }
 
@@ -465,16 +461,14 @@ export default class RuntimeResolverImpl implements RuntimeResolver<OwnedTemplat
       assert(`missing component class ${name}`, pair.component.class !== undefined);
 
       let ComponentClass = pair.component.class!;
-      let wrapper = getManager(ComponentClass);
+      let manager = getComponentManager(owner, ComponentClass);
 
-      if (wrapper !== null && wrapper.type === 'component') {
-        let { factory } = wrapper;
-
-        if (wrapper.internal) {
+      if (manager !== undefined) {
+        if (isInternalManager(manager)) {
           assert(`missing layout for internal component ${name}`, pair.layout !== null);
 
           definition = new InternalComponentDefinition(
-            factory(owner) as InternalComponentManager,
+            manager,
             ComponentClass as typeof InternalComponent,
             layout!
           );
@@ -482,7 +476,7 @@ export default class RuntimeResolverImpl implements RuntimeResolver<OwnedTemplat
           definition = new CustomManagerDefinition(
             name,
             pair.component,
-            factory(owner) as ManagerDelegate<unknown>,
+            manager,
             layout !== undefined
               ? layout
               : owner.lookup<TemplateFactory>(P`template:components/-default`)!(owner)
diff --git a/packages/@ember/-internals/glimmer/lib/utils/custom-component-manager.ts b/packages/@ember/-internals/glimmer/lib/utils/custom-component-manager.ts
deleted file mode 100644
index e4ab147e8cc..00000000000
--- a/packages/@ember/-internals/glimmer/lib/utils/custom-component-manager.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import { Owner } from '@ember/-internals/owner';
-import { deprecate } from '@ember/debug';
-import { COMPONENT_MANAGER_STRING_LOOKUP } from '@ember/deprecated-features';
-import { ManagerDelegate } from '../component-managers/custom';
-import { getManager, ManagerFactory, setManager } from './managers';
-
-export function setComponentManager(
-  stringOrFunction: string | ManagerFactory<ManagerDelegate<unknown>>,
-  obj: any
-) {
-  let factory: ManagerFactory<unknown>;
-  if (COMPONENT_MANAGER_STRING_LOOKUP && typeof stringOrFunction === 'string') {
-    deprecate(
-      'Passing the name of the component manager to "setupComponentManager" is deprecated. Please pass a function that produces an instance of the manager.',
-      false,
-      {
-        id: 'deprecate-string-based-component-manager',
-        until: '4.0.0',
-        url: 'https://emberjs.com/deprecations/v3.x/#toc_component-manager-string-lookup',
-      }
-    );
-    factory = function(owner: Owner) {
-      return owner.lookup(`component-manager:${stringOrFunction}`);
-    };
-  } else {
-    factory = stringOrFunction as ManagerFactory<unknown>;
-  }
-
-  return setManager({ factory, internal: false, type: 'component' }, obj);
-}
-
-export function getComponentManager<T>(obj: any): undefined | ManagerFactory<ManagerDelegate<T>> {
-  let wrapper = getManager<ManagerDelegate<T>>(obj);
-
-  if (wrapper && !wrapper.internal && wrapper.type === 'component') {
-    return wrapper.factory;
-  } else {
-    return undefined;
-  }
-}
diff --git a/packages/@ember/-internals/glimmer/lib/utils/custom-modifier-manager.ts b/packages/@ember/-internals/glimmer/lib/utils/custom-modifier-manager.ts
deleted file mode 100644
index 8a31a370720..00000000000
--- a/packages/@ember/-internals/glimmer/lib/utils/custom-modifier-manager.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-import { getManager, ManagerFactory, setManager } from './managers';
-
-export function setModifierManager(factory: ManagerFactory<unknown>, obj: any) {
-  return setManager({ factory, internal: false, type: 'modifier' }, obj);
-}
-
-export function getModifierManager<T>(obj: any): undefined | ManagerFactory<T> {
-  let wrapper = getManager<T>(obj);
-
-  if (wrapper && !wrapper.internal && wrapper.type === 'modifier') {
-    return wrapper.factory;
-  } else {
-    return undefined;
-  }
-}
diff --git a/packages/@ember/-internals/glimmer/lib/utils/managers.ts b/packages/@ember/-internals/glimmer/lib/utils/managers.ts
index 9abdd6d78f2..b89ea86ae07 100644
--- a/packages/@ember/-internals/glimmer/lib/utils/managers.ts
+++ b/packages/@ember/-internals/glimmer/lib/utils/managers.ts
@@ -1,34 +1,144 @@
 import { Owner } from '@ember/-internals/owner';
-import { Option } from '@glimmer/interfaces';
+import { deprecate } from '@ember/debug';
+import { COMPONENT_MANAGER_STRING_LOOKUP } from '@ember/deprecated-features';
+import { ManagerDelegate as ComponentManagerDelegate } from '../component-managers/custom';
+import InternalComponentManager from '../component-managers/internal';
+import { ModifierManagerDelegate } from '../modifiers/custom';
 
-const MANAGERS: WeakMap<object, ManagerWrapper<unknown>> = new WeakMap();
+type ManagerDelegate =
+  | ComponentManagerDelegate<unknown>
+  | InternalComponentManager
+  | ModifierManagerDelegate<unknown>;
 
-const getPrototypeOf = Object.getPrototypeOf;
+const COMPONENT_MANAGERS = new WeakMap<
+  object,
+  ManagerFactory<ComponentManagerDelegate<unknown> | InternalComponentManager>
+>();
 
-export type ManagerFactory<ManagerDelegate> = (owner: Owner) => ManagerDelegate;
+const MODIFIER_MANAGERS = new WeakMap<object, ManagerFactory<ModifierManagerDelegate<unknown>>>();
 
-export interface ManagerWrapper<ManagerDelegate> {
-  factory: ManagerFactory<ManagerDelegate>;
-  internal: boolean;
-  type: 'component' | 'modifier';
-}
+const MANAGER_INSTANCES: WeakMap<Owner, WeakMap<ManagerFactory, unknown>> = new WeakMap();
+
+export type ManagerFactory<D extends ManagerDelegate = ManagerDelegate> = (owner: Owner) => D;
+
+///////////
 
-export function setManager<ManagerDelegate>(wrapper: ManagerWrapper<ManagerDelegate>, obj: object) {
-  MANAGERS.set(obj, wrapper);
+const getPrototypeOf = Object.getPrototypeOf;
+
+function setManager<Def extends object>(
+  map: WeakMap<object, ManagerFactory>,
+  factory: ManagerFactory,
+  obj: Def
+): Def {
+  map.set(obj, factory);
   return obj;
 }
 
-export function getManager<ManagerDelegate>(obj: object): Option<ManagerWrapper<ManagerDelegate>> {
+function getManager<D extends ManagerDelegate>(
+  map: WeakMap<object, ManagerFactory<D>>,
+  obj: object
+): ManagerFactory<D> | undefined {
   let pointer = obj;
   while (pointer !== undefined && pointer !== null) {
-    let manager = MANAGERS.get(pointer);
+    const manager = map.get(pointer);
 
     if (manager !== undefined) {
-      return manager as ManagerWrapper<ManagerDelegate>;
+      return manager;
     }
 
     pointer = getPrototypeOf(pointer);
   }
 
-  return null;
+  return undefined;
+}
+
+function getManagerInstanceForOwner<D extends ManagerDelegate>(
+  owner: Owner,
+  factory: ManagerFactory<D>
+): D {
+  let managers = MANAGER_INSTANCES.get(owner);
+
+  if (managers === undefined) {
+    managers = new WeakMap();
+    MANAGER_INSTANCES.set(owner, managers);
+  }
+
+  let instance = managers.get(factory);
+
+  if (instance === undefined) {
+    instance = factory(owner);
+    managers.set(factory, instance!);
+  }
+
+  // We know for sure that it's the correct type at this point, but TS can't know
+  return instance as D;
+}
+
+///////////
+
+export function setModifierManager(
+  factory: ManagerFactory<ModifierManagerDelegate<unknown>>,
+  definition: object
+) {
+  return setManager(MODIFIER_MANAGERS, factory, definition);
+}
+
+export function getModifierManager(
+  owner: Owner,
+  definition: object
+): ModifierManagerDelegate<unknown> | undefined {
+  const factory = getManager(MODIFIER_MANAGERS, definition);
+
+  if (factory !== undefined) {
+    return getManagerInstanceForOwner(owner, factory);
+  }
+
+  return undefined;
+}
+
+export function setComponentManager(
+  stringOrFunction:
+    | string
+    | ManagerFactory<ComponentManagerDelegate<unknown> | InternalComponentManager>,
+  obj: object
+) {
+  let factory: ManagerFactory<ComponentManagerDelegate<unknown> | InternalComponentManager>;
+  if (COMPONENT_MANAGER_STRING_LOOKUP && typeof stringOrFunction === 'string') {
+    deprecate(
+      'Passing the name of the component manager to "setupComponentManager" is deprecated. Please pass a function that produces an instance of the manager.',
+      false,
+      {
+        id: 'deprecate-string-based-component-manager',
+        until: '4.0.0',
+        url: 'https://emberjs.com/deprecations/v3.x/#toc_component-manager-string-lookup',
+      }
+    );
+    factory = function(owner: Owner) {
+      return owner.lookup<ComponentManagerDelegate<unknown> | InternalComponentManager>(
+        `component-manager:${stringOrFunction}`
+      )!;
+    };
+  } else {
+    factory = stringOrFunction as ManagerFactory<
+      ComponentManagerDelegate<unknown> | InternalComponentManager
+    >;
+  }
+
+  return setManager(COMPONENT_MANAGERS, factory, obj);
+}
+
+export function getComponentManager(
+  owner: Owner,
+  definition: object
+): ComponentManagerDelegate<unknown> | InternalComponentManager | undefined {
+  const factory = getManager<ComponentManagerDelegate<unknown> | InternalComponentManager>(
+    COMPONENT_MANAGERS,
+    definition
+  );
+
+  if (factory !== undefined) {
+    return getManagerInstanceForOwner(owner, factory);
+  }
+
+  return undefined;
 }