From 4229aa7aeec516c6e8ddbb6d2910076d0b6b83b2 Mon Sep 17 00:00:00 2001
From: Martin Nabhan <martin@impact-blue.co.jp>
Date: Wed, 30 Mar 2022 19:44:59 +0900
Subject: [PATCH] fix: multi-byte-streaming-ssr
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When using streaming SSR decodeText is called repeatedly with incoming
chunks of data. In that case a multi-byte character may occasionally
split between chunks, causing corruption. By setting the TextDecoder
option 'stream' to true, and reusing the same TextDecoder instance,
TextDecoder will memorise “unfinished” characters and decode them when
the next chunk comes.
---
 packages/next/server/node-web-streams-helper.ts   |  8 +++++---
 .../react-18-streaming-ssr/index.test.ts          | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/packages/next/server/node-web-streams-helper.ts b/packages/next/server/node-web-streams-helper.ts
index f854b4e675f3..c3e4eb636b04 100644
--- a/packages/next/server/node-web-streams-helper.ts
+++ b/packages/next/server/node-web-streams-helper.ts
@@ -111,8 +111,8 @@ export function encodeText(input: string) {
   return new TextEncoder().encode(input)
 }
 
-export function decodeText(input?: Uint8Array) {
-  return new TextDecoder().decode(input)
+export function decodeText(input?: Uint8Array, textDecoder?: TextDecoder) {
+  return textDecoder ? textDecoder.decode(input, { stream: true } ) : new TextDecoder().decode(input)
 }
 
 export function createTransformStream<Input, Output>({
@@ -207,9 +207,11 @@ export function createBufferedTransformStream(): TransformStream<
     return pendingFlush
   }
 
+  const textDecoder = new TextDecoder();
+
   return createTransformStream({
     transform(chunk, controller) {
-      bufferedString += decodeText(chunk)
+      bufferedString += decodeText(chunk, textDecoder)
       flushBuffer(controller)
     },
 
diff --git a/test/production/react-18-streaming-ssr/index.test.ts b/test/production/react-18-streaming-ssr/index.test.ts
index 93a9a873051f..19994e45b2ad 100644
--- a/test/production/react-18-streaming-ssr/index.test.ts
+++ b/test/production/react-18-streaming-ssr/index.test.ts
@@ -61,6 +61,15 @@ describe('react 18 streaming SSR with custom next configs', () => {
             return <p>hello nextjs</p>
           }
         `,
+        'pages/multi-byte.js': `
+          export default function Page() {
+            return (
+              <div>
+                <p>{"マルチバイト".repeat(28)}</p>
+              </div>
+            );
+          }
+        `,
       },
       nextConfig: {
         trailingSlash: true,
@@ -98,4 +107,10 @@ describe('react 18 streaming SSR with custom next configs', () => {
     expect(res.status).toBe(200)
     expect(html).toContain('hello nextjs')
   })
+
+  it('should render multi-byte characters correctly in streaming', async () => {
+    const html = await renderViaHTTP(next.url, '/multi-byte')
+    console.log(html)
+    expect(html).toContain('マルチバイト'.repeat(28))
+  })
 })