diff --git a/package.json b/package.json
index 2629fc434..9997896d5 100644
--- a/package.json
+++ b/package.json
@@ -176,6 +176,7 @@
       "node/file-extension-in-import": "off",
       "react/prop-types": "off",
       "unicorn/no-await-expression-member": "off",
+      "unicorn/prefer-code-point": "off",
       "unicorn/prefer-node-protocol": "off",
       "capitalized-comments": "off",
       "complexity": "off",
diff --git a/packages/mdx/lib/plugin/recma-jsx-rewrite.js b/packages/mdx/lib/plugin/recma-jsx-rewrite.js
index 298017ab6..db2e16110 100644
--- a/packages/mdx/lib/plugin/recma-jsx-rewrite.js
+++ b/packages/mdx/lib/plugin/recma-jsx-rewrite.js
@@ -180,11 +180,10 @@ export function recmaJsxRewrite(options = {}) {
               fnScope.tags.push(id)
             }
 
-            node.openingElement.name = {
-              type: 'JSXMemberExpression',
-              object: {type: 'JSXIdentifier', name: '_components'},
-              property: name
-            }
+            node.openingElement.name = toJsxIdOrMemberExpression([
+              '_components',
+              id
+            ])
 
             if (node.closingElement) {
               node.closingElement.name = toJsxIdOrMemberExpression([
@@ -224,7 +223,9 @@ export function recmaJsxRewrite(options = {}) {
             defaults.push({
               type: 'Property',
               kind: 'init',
-              key: {type: 'Identifier', name},
+              key: isIdentifierName(name)
+                ? {type: 'Identifier', name}
+                : {type: 'Literal', value: name},
               value: {type: 'Literal', value: name},
               method: false,
               shorthand: false,
diff --git a/packages/mdx/lib/util/estree-util-to-id-or-member-expression.js b/packages/mdx/lib/util/estree-util-to-id-or-member-expression.js
index 0d268453e..63fe3f1fd 100644
--- a/packages/mdx/lib/util/estree-util-to-id-or-member-expression.js
+++ b/packages/mdx/lib/util/estree-util-to-id-or-member-expression.js
@@ -6,23 +6,35 @@
  * @typedef {import('estree-jsx').JSXMemberExpression} JSXMemberExpression
  */
 
-import {name as isIdentifierName} from 'estree-util-is-identifier-name'
+import {
+  start as esStart,
+  cont as esCont,
+  name as isIdentifierName
+} from 'estree-util-is-identifier-name'
 
 export const toIdOrMemberExpression = toIdOrMemberExpressionFactory(
   'Identifier',
-  'MemberExpression'
+  'MemberExpression',
+  isIdentifierName
 )
 
 export const toJsxIdOrMemberExpression =
   // @ts-expect-error: fine
   /** @type {(ids: Array<string|number>) => JSXIdentifier|JSXMemberExpression)} */
-  (toIdOrMemberExpressionFactory('JSXIdentifier', 'JSXMemberExpression'))
+  (
+    toIdOrMemberExpressionFactory(
+      'JSXIdentifier',
+      'JSXMemberExpression',
+      isJsxIdentifierName
+    )
+  )
 
 /**
- * @param {string} [idType]
- * @param {string} [memberType]
+ * @param {string} idType
+ * @param {string} memberType
+ * @param {(value: string) => boolean} isIdentifier
  */
-function toIdOrMemberExpressionFactory(idType, memberType) {
+function toIdOrMemberExpressionFactory(idType, memberType, isIdentifier) {
   return toIdOrMemberExpression
   /**
    * @param {Array<string|number>} ids
@@ -35,12 +47,18 @@ function toIdOrMemberExpressionFactory(idType, memberType) {
 
     while (++index < ids.length) {
       const name = ids[index]
+      const valid = typeof name === 'string' && isIdentifier(name)
+
+      // A value of `asd.123` could be turned into `asd['123']` in the JS form,
+      // but JSX does not have a form for it, so throw.
+      /* c8 ignore next 3 */
+      if (idType === 'JSXIdentifier' && !valid) {
+        throw new Error('Cannot turn `' + name + '` into a JSX identifier')
+      }
+
       /** @type {Identifier|Literal} */
       // @ts-expect-error: JSX is fine.
-      const id =
-        typeof name === 'string' && isIdentifierName(name)
-          ? {type: idType, name}
-          : {type: 'Literal', value: name}
+      const id = valid ? {type: idType, name} : {type: 'Literal', value: name}
       // @ts-expect-error: JSX is fine.
       object = object
         ? {
@@ -62,3 +80,29 @@ function toIdOrMemberExpressionFactory(idType, memberType) {
     return object
   }
 }
+
+/**
+ * Checks if the given string is a valid JSX identifier name.
+ * @param {string} name
+ */
+function isJsxIdentifierName(name) {
+  let index = -1
+
+  while (++index < name.length) {
+    // We currently receive valid input, but this catches bugs and is needed
+    // when externalized.
+    /* c8 ignore next */
+    if (!(index ? jsxCont : esStart)(name.charCodeAt(index))) return false
+  }
+
+  // `false` if `name` is empty.
+  return index > 0
+}
+
+/**
+ * Checks if the given character code can continue a JSX identifier.
+ * @param {number} code
+ */
+function jsxCont(code) {
+  return code === 45 /* `-` */ || esCont(code)
+}
diff --git a/packages/mdx/test/compile.js b/packages/mdx/test/compile.js
index 5e85d9ae8..51307f605 100644
--- a/packages/mdx/test/compile.js
+++ b/packages/mdx/test/compile.js
@@ -134,6 +134,22 @@ test('compile', async () => {
     'should compile a non-element document (rehype, single element)'
   )
 
+  assert.equal(
+    renderToStaticMarkup(
+      React.createElement(
+        await run(
+          compileSync('y', {
+            rehypePlugins: [
+              () => () => ({type: 'element', tagName: 'a-b', children: []})
+            ]
+          })
+        )
+      )
+    ),
+    '<a-b></a-b>',
+    'should compile custom elements'
+  )
+
   assert.equal(
     renderToStaticMarkup(
       React.createElement(
@@ -798,6 +814,26 @@ test('jsx', async () => {
     'should serialize fragments, expressions'
   )
 
+  assert.equal(
+    String(compileSync('{<a-b></a-b>}', {jsx: true})),
+    [
+      '/*@jsxRuntime automatic @jsxImportSource react*/',
+      'function MDXContent(props = {}) {',
+      '  const {wrapper: MDXLayout} = props.components || ({});',
+      '  return MDXLayout ? <MDXLayout {...props}><_createMdxContent /></MDXLayout> : _createMdxContent();',
+      '  function _createMdxContent() {',
+      '    const _components = Object.assign({',
+      '      "a-b": "a-b"',
+      '    }, props.components);',
+      '    return <>{<_components.a-b></_components.a-b>}</>;',
+      '  }',
+      '}',
+      'export default MDXContent;',
+      ''
+    ].join('\n'),
+    'should serialize custom elements inside expressions'
+  )
+
   assert.equal(
     String(compileSync('Hello {props.name}', {jsx: true})),
     [