From 04cf4c974f1f759934506c58f0ae98835b75ba17 Mon Sep 17 00:00:00 2001
From: Martin Nabhan <martin@impact-blue.co.jp>
Date: Wed, 30 Mar 2022 19:44:59 +0900
Subject: [PATCH] fix: multi-byte-streaming-ssr
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When using streaming SSR decodeText is called repeatedly with incoming
chunks of data. In that case a multi-byte character may occasionally
split between chunks, causing corruption. By setting the TextDecoder
option 'stream' to true, and reusing the same TextDecoder instance,
TextDecoder will memorise “unfinished” characters and decode them when
the next chunk comes.
---
 packages/next/server/node-web-streams-helper.ts    | 10 +++++++---
 .../react-18-streaming-ssr/index.test.ts           | 14 ++++++++++++++
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/packages/next/server/node-web-streams-helper.ts b/packages/next/server/node-web-streams-helper.ts
index f854b4e675f..f4fe2632ed9 100644
--- a/packages/next/server/node-web-streams-helper.ts
+++ b/packages/next/server/node-web-streams-helper.ts
@@ -111,8 +111,10 @@ export function encodeText(input: string) {
   return new TextEncoder().encode(input)
 }
 
-export function decodeText(input?: Uint8Array) {
-  return new TextDecoder().decode(input)
+export function decodeText(input?: Uint8Array, textDecoder?: TextDecoder) {
+  return textDecoder
+    ? textDecoder.decode(input, { stream: true })
+    : new TextDecoder().decode(input)
 }
 
 export function createTransformStream<Input, Output>({
@@ -207,9 +209,11 @@ export function createBufferedTransformStream(): TransformStream<
     return pendingFlush
   }
 
+  const textDecoder = new TextDecoder()
+
   return createTransformStream({
     transform(chunk, controller) {
-      bufferedString += decodeText(chunk)
+      bufferedString += decodeText(chunk, textDecoder)
       flushBuffer(controller)
     },
 
diff --git a/test/production/react-18-streaming-ssr/index.test.ts b/test/production/react-18-streaming-ssr/index.test.ts
index 93a9a873051..52aaaccc55a 100644
--- a/test/production/react-18-streaming-ssr/index.test.ts
+++ b/test/production/react-18-streaming-ssr/index.test.ts
@@ -61,6 +61,15 @@ describe('react 18 streaming SSR with custom next configs', () => {
             return <p>hello nextjs</p>
           }
         `,
+        'pages/multi-byte.js': `
+          export default function Page() {
+            return (
+              <div>
+                <p>{"マルチバイト".repeat(28)}</p>
+              </div>
+            );
+          }
+        `,
       },
       nextConfig: {
         trailingSlash: true,
@@ -98,4 +107,9 @@ describe('react 18 streaming SSR with custom next configs', () => {
     expect(res.status).toBe(200)
     expect(html).toContain('hello nextjs')
   })
+
+  it('should render multi-byte characters correctly in streaming', async () => {
+    const html = await renderViaHTTP(next.url, '/multi-byte')
+    expect(html).toContain('マルチバイト'.repeat(28))
+  })
 })