diff --git a/errors/babel-font-loader-conflict.md b/errors/babel-font-loader-conflict.md
index 6999883ddcda..00fb535e0b86 100644
--- a/errors/babel-font-loader-conflict.md
+++ b/errors/babel-font-loader-conflict.md
@@ -1,8 +1,8 @@
-# Babel and Font loader conflict
+# Babel and `@next/font` conflict
 
 #### Why This Error Occurred
 
-You have tried to use `experimental.fontLoaders` with a custom babel config. When your application has a custom babel config you opt-out of the Next.js Compiler which is required to use `experimental.fontLoaders`.
+You have tried to use `@next/font` with a custom babel config. When your application has a custom babel config you opt-out of the Next.js Compiler which is required to use `@next/font`.
 
 #### Possible Ways to Fix It
 
diff --git a/packages/next/build/babel/loader/get-config.ts b/packages/next/build/babel/loader/get-config.ts
index f74191d38e49..92e7afcf1eea 100644
--- a/packages/next/build/babel/loader/get-config.ts
+++ b/packages/next/build/babel/loader/get-config.ts
@@ -121,6 +121,10 @@ function getPlugins(
         { type: 'plugin' }
       )
     : null
+  const nextFontUnsupported = createConfigItem(
+    [require('../plugins/next-font-unsupported')],
+    { type: 'plugin' }
+  )
 
   return [
     reactRefreshItem,
@@ -130,6 +134,7 @@ function getPlugins(
     transformDefineItem,
     nextSsgItem,
     commonJsItem,
+    nextFontUnsupported,
   ].filter(Boolean)
 }
 
diff --git a/packages/next/build/babel/plugins/next-font-unsupported.ts b/packages/next/build/babel/plugins/next-font-unsupported.ts
new file mode 100644
index 000000000000..dd17ebf551f1
--- /dev/null
+++ b/packages/next/build/babel/plugins/next-font-unsupported.ts
@@ -0,0 +1,23 @@
+import { NodePath, PluginObj, types } from 'next/dist/compiled/babel/core'
+
+export default function NextPageDisallowReExportAllExports(): PluginObj<any> {
+  return {
+    visitor: {
+      ImportDeclaration(path: NodePath<types.ImportDeclaration>) {
+        if (
+          ['@next/font/local', '@next/font/google'].includes(
+            path.node.source.value
+          )
+        ) {
+          const err = new SyntaxError(
+            `"@next/font" requires SWC although Babel is being used due to a custom babel config being present.\nRead more: https://nextjs.org/docs/messages/babel-font-loader-conflict`
+          )
+          ;(err as any).code = 'BABEL_PARSE_ERROR'
+          ;(err as any).loc =
+            path.node.loc?.start ?? path.node.loc?.end ?? path.node.loc
+          throw err
+        }
+      },
+    },
+  }
+}
diff --git a/packages/next/build/webpack-config.ts b/packages/next/build/webpack-config.ts
index e49a93a0c8f8..050ee856dae7 100644
--- a/packages/next/build/webpack-config.ts
+++ b/packages/next/build/webpack-config.ts
@@ -662,16 +662,6 @@ export default async function getBaseWebpackConfig(
     loggedIgnoredCompilerOptions = true
   }
 
-  if (!useSWCLoader && babelConfigFile && config.experimental.fontLoaders) {
-    Log.error(
-      `"experimental.fontLoaders" is enabled which requires SWC although Babel is being used due to custom babel config being present "${path.relative(
-        dir,
-        babelConfigFile
-      )}".\nSee more info here: https://nextjs.org/docs/messages/babel-font-loader-conflict`
-    )
-    process.exit(1)
-  }
-
   const getBabelLoader = () => {
     return {
       loader: require.resolve('./babel/loader/index'),
diff --git a/packages/next/build/webpack/config/blocks/css/index.ts b/packages/next/build/webpack/config/blocks/css/index.ts
index 3f4d9151ccf7..602e4b860a7a 100644
--- a/packages/next/build/webpack/config/blocks/css/index.ts
+++ b/packages/next/build/webpack/config/blocks/css/index.ts
@@ -11,7 +11,6 @@ import {
   getGlobalModuleImportError,
   getLocalModuleImportError,
   getFontLoaderDocumentImportError,
-  getFontLoaderImportError,
 } from './messages'
 import { getPostCssPlugins } from './plugins'
 
@@ -25,8 +24,6 @@ const regexCssModules = /\.module\.css$/
 // RegExps for Syntactically Awesome Style Sheets
 const regexSassGlobal = /(?<!\.module)\.(scss|sass)$/
 const regexSassModules = /\.module\.(scss|sass)$/
-// Also match the virtual client entry which doesn't have file path
-const regexClientEntry = /^$/
 
 /**
  * Mark a rule as removable if built-in CSS support is disabled
@@ -218,35 +215,11 @@ export const css = curry(async function css(
           markRemovable({
             sideEffects: false,
             test: fontLoaderPath,
-            issuer: {
-              and: [
-                {
-                  or: [ctx.rootDirectory, regexClientEntry],
-                },
-              ],
-              not: [/node_modules/],
-            },
             use: getFontLoader(ctx, lazyPostCSSInitializer, fontLoaderOptions),
           }),
         ],
       })
     )
-
-    fns.push(
-      loader({
-        oneOf: [
-          markRemovable({
-            test: fontLoaderPath,
-            use: {
-              loader: 'error-loader',
-              options: {
-                reason: getFontLoaderImportError(),
-              },
-            },
-          }),
-        ],
-      })
-    )
   })
 
   // CSS cannot be imported in _document. This comes before everything because
diff --git a/packages/next/build/webpack/config/blocks/css/messages.ts b/packages/next/build/webpack/config/blocks/css/messages.ts
index b6b1b5614308..f0848c817ae6 100644
--- a/packages/next/build/webpack/config/blocks/css/messages.ts
+++ b/packages/next/build/webpack/config/blocks/css/messages.ts
@@ -37,9 +37,3 @@ export function getFontLoaderDocumentImportError() {
     'cannot'
   )} be used within ${chalk.cyan('pages/_document.js')}.`
 }
-
-export function getFontLoaderImportError() {
-  return `Font loader error:\nFont loaders ${chalk.bold(
-    'cannot'
-  )} be used from within ${chalk.bold('node_modules')}.`
-}
diff --git a/packages/next/server/config.ts b/packages/next/server/config.ts
index aabba5567627..4e53739ee8a4 100644
--- a/packages/next/server/config.ts
+++ b/packages/next/server/config.ts
@@ -24,7 +24,6 @@ import {
 } from '../shared/lib/image-config'
 import { loadEnvConfig } from '@next/env'
 import { gte as semverGte } from 'next/dist/compiled/semver'
-import { getDependencies } from '../lib/get-package-version'
 
 export { DomainLocale, NextConfig, normalizeConfig } from './config-shared'
 
@@ -93,15 +92,17 @@ export function setHttpClientAndAgentOptions(config: {
   )
 }
 
-async function setFontLoaderDefaults(config: NextConfigComplete, dir: string) {
-  // Add @next/font loaders by default if they're installed
-  const hasNextFontDependency = (
-    await getDependencies({
-      cwd: dir,
-    })
-  ).dependencies['@next/font']
+function setFontLoaderDefaults(config: NextConfigComplete) {
+  try {
+    // eslint-disable-next-line import/no-extraneous-dependencies
+    const nextFontVersion = require('@next/font/package.json').version
+    const nextVersion = require('next/package.json').version
+    if (nextFontVersion !== nextVersion) {
+      Log.warn(
+        `Different versions of @next/font (${nextFontVersion}) and next (${nextVersion}) detected. This may lead to unexpected behavior.`
+      )
+    }
 
-  if (hasNextFontDependency) {
     const googleFontLoader = {
       loader: '@next/font/google',
     }
@@ -128,7 +129,7 @@ async function setFontLoaderDefaults(config: NextConfigComplete, dir: string) {
     ) {
       config.experimental.fontLoaders.push(localFontLoader)
     }
-  }
+  } catch {}
 }
 
 function assignDefaults(dir: string, userConfig: { [key: string]: any }) {
@@ -906,7 +907,7 @@ export default async function loadConfig(
       configFileName,
       ...userConfig,
     }) as NextConfigComplete
-    await setFontLoaderDefaults(completeConfig, dir)
+    setFontLoaderDefaults(completeConfig)
     return completeConfig
   } else {
     const configBaseName = basename(CONFIG_FILES[0], extname(CONFIG_FILES[0]))
@@ -936,6 +937,6 @@ export default async function loadConfig(
   ) as NextConfigComplete
   completeConfig.configFileName = configFileName
   setHttpClientAndAgentOptions(completeConfig)
-  await setFontLoaderDefaults(completeConfig, dir)
+  setFontLoaderDefaults(completeConfig)
   return completeConfig
 }
diff --git a/test/e2e/app-dir/rsc-external.test.ts b/test/e2e/app-dir/rsc-external.test.ts
index 290bcecffd53..7eb2160ea56e 100644
--- a/test/e2e/app-dir/rsc-external.test.ts
+++ b/test/e2e/app-dir/rsc-external.test.ts
@@ -30,6 +30,7 @@ describe('app dir - rsc external dependency', () => {
     next = await createNext({
       files: new FileRef(path.join(__dirname, './rsc-external')),
       dependencies: {
+        '@next/font': 'canary',
         react: 'latest',
         'react-dom': 'latest',
         swr: '2.0.0-rc.0',
@@ -155,4 +156,14 @@ describe('app dir - rsc external dependency', () => {
       )
     ).toBe('rgb(255, 0, 0)')
   })
+
+  it('should handle external @next/font', async () => {
+    const browser = await webdriver(next.url, '/font')
+
+    expect(
+      await browser.eval(
+        `window.getComputedStyle(document.querySelector('p')).fontFamily`
+      )
+    ).toMatch(/^__myFont_.{6}, __myFont_Fallback_.{6}$/)
+  })
 })
diff --git a/test/e2e/app-dir/rsc-external/app/font/page.js b/test/e2e/app-dir/rsc-external/app/font/page.js
new file mode 100644
index 000000000000..2ffe578183c4
--- /dev/null
+++ b/test/e2e/app-dir/rsc-external/app/font/page.js
@@ -0,0 +1,5 @@
+import { myFont } from 'font'
+
+export default function Page() {
+  return <p className={myFont.className}>Hello world</p>
+}
diff --git a/test/e2e/app-dir/rsc-external/next.config.js b/test/e2e/app-dir/rsc-external/next.config.js
index ac3e4ea3960b..8e2dcc5de939 100644
--- a/test/e2e/app-dir/rsc-external/next.config.js
+++ b/test/e2e/app-dir/rsc-external/next.config.js
@@ -3,6 +3,6 @@ module.exports = {
   experimental: {
     appDir: true,
     serverComponentsExternalPackages: ['conditional-exports-optout'],
-    transpilePackages: ['untranspiled-module', 'css'],
+    transpilePackages: ['untranspiled-module', 'css', 'font'],
   },
 }
diff --git a/test/e2e/app-dir/rsc-external/node_modules_bak/font/index.ts b/test/e2e/app-dir/rsc-external/node_modules_bak/font/index.ts
new file mode 100644
index 000000000000..5a31d967538f
--- /dev/null
+++ b/test/e2e/app-dir/rsc-external/node_modules_bak/font/index.ts
@@ -0,0 +1,3 @@
+import localFont from '@next/font/local'
+
+export const myFont = localFont({ src: './my-font.woff2' })
diff --git a/test/e2e/app-dir/rsc-external/node_modules_bak/font/my-font.woff2 b/test/e2e/app-dir/rsc-external/node_modules_bak/font/my-font.woff2
new file mode 100644
index 000000000000..a6b3c3a9d69f
Binary files /dev/null and b/test/e2e/app-dir/rsc-external/node_modules_bak/font/my-font.woff2 differ
diff --git a/test/e2e/app-dir/rsc-external/node_modules_bak/font/package.json b/test/e2e/app-dir/rsc-external/node_modules_bak/font/package.json
new file mode 100644
index 000000000000..7b73363d359e
--- /dev/null
+++ b/test/e2e/app-dir/rsc-external/node_modules_bak/font/package.json
@@ -0,0 +1,4 @@
+{
+  "name": "font",
+  "main": "index.ts"
+}
diff --git a/test/e2e/next-font/babel/next.config.js b/test/e2e/next-font/babel/next.config.js
deleted file mode 100644
index 3f4a776b7f40..000000000000
--- a/test/e2e/next-font/babel/next.config.js
+++ /dev/null
@@ -1,10 +0,0 @@
-module.exports = {
-  experimental: {
-    fontLoaders: [
-      {
-        loader: '@next/font/google',
-        options: { subsets: ['latin'] },
-      },
-    ],
-  },
-}
diff --git a/test/e2e/next-font/babel.test.ts b/test/production/next-font/babel-unsupported.test.ts
similarity index 59%
rename from test/e2e/next-font/babel.test.ts
rename to test/production/next-font/babel-unsupported.test.ts
index d5ff9d6511ce..1214ef377b57 100644
--- a/test/e2e/next-font/babel.test.ts
+++ b/test/production/next-font/babel-unsupported.test.ts
@@ -2,19 +2,13 @@ import { createNext, FileRef } from 'e2e-utils'
 import { NextInstance } from 'test/lib/next-modes/base'
 import { join } from 'path'
 
-describe('@next/font/google babel', () => {
-  const isNextStart = (global as any).isNextStart
+describe('@next/fon babel unsupported', () => {
   let next: NextInstance
 
-  if (!isNextStart) {
-    it('should only run on next start', () => {})
-    return
-  }
-
   beforeAll(async () => {
     next = await createNext({
       skipStart: true,
-      files: new FileRef(join(__dirname, 'babel')),
+      files: new FileRef(join(__dirname, 'babel-unsupported')),
       dependencies: {
         '@next/font': 'canary',
       },
@@ -27,7 +21,7 @@ describe('@next/font/google babel', () => {
       'next build failed with code/signal 1'
     )
     expect(next.cliOutput).toMatch(
-      /"experimental.fontLoaders" is enabled which requires SWC although Babel is being used due to custom babel config being present ".babelrc"./
+      /"@next\/font" requires SWC although Babel is being used due to a custom babel config being present./
     )
   })
 })
diff --git a/test/e2e/next-font/babel/.babelrc b/test/production/next-font/babel-unsupported/.babelrc
similarity index 100%
rename from test/e2e/next-font/babel/.babelrc
rename to test/production/next-font/babel-unsupported/.babelrc
diff --git a/test/e2e/next-font/babel/pages/index.js b/test/production/next-font/babel-unsupported/pages/index.js
similarity index 50%
rename from test/e2e/next-font/babel/pages/index.js
rename to test/production/next-font/babel-unsupported/pages/index.js
index a681aa7ce257..a5942d198ea3 100644
--- a/test/e2e/next-font/babel/pages/index.js
+++ b/test/production/next-font/babel-unsupported/pages/index.js
@@ -1,3 +1,7 @@
+import { Inter } from '@next/font/google'
+
+console.log(Inter)
+
 export default function Page() {
   return <p>Hello world</p>
 }