fix: multi-byte-streaming-ssr

When using streaming SSR decodeText is called repeatedly with incoming
chunks of data. In that case a multi-byte character may occasionally
split between chunks, causing corruption. By setting the TextDecoder
option 'stream' to true, and reusing the same TextDecoder instance,
TextDecoder will memorise “unfinished” characters and decode them when
the next chunk comes.

packages/next/server/node-web-streams-helper.ts

@@ -111,8 +111,8 @@ export function encodeText(input: string) {
   return new TextEncoder().encode(input)
 }
 
-export function decodeText(input?: Uint8Array) {
-  return new TextDecoder().decode(input)
+export function decodeText(input?: Uint8Array, textDecoder?: TextDecoder) {
+  return textDecoder ? textDecoder.decode(input, { stream: true } ) : new TextDecoder().decode(input)
 }
 
 export function createTransformStream<Input, Output>({
@@ -207,9 +207,11 @@ export function createBufferedTransformStream(): TransformStream<
     return pendingFlush
   }
 
+  const textDecoder = new TextDecoder();
+
   return createTransformStream({
     transform(chunk, controller) {
-      bufferedString += decodeText(chunk)
+      bufferedString += decodeText(chunk, textDecoder)
       flushBuffer(controller)
     },
 

test/production/react-18-streaming-ssr/index.test.ts

@@ -61,6 +61,15 @@ describe('react 18 streaming SSR with custom next configs', () => {
             return <p>hello nextjs</p>
           }
         `,
+        'pages/multi-byte.js': `
+          export default function Page() {
+            return (
+              <div>
+                <p>{"マルチバイト".repeat(28)}</p>
+              </div>
+            );
+          }
+        `,
       },
       nextConfig: {
         trailingSlash: true,
@@ -98,4 +107,9 @@ describe('react 18 streaming SSR with custom next configs', () => {
     expect(res.status).toBe(200)
     expect(html).toContain('hello nextjs')
   })
+
+  it('should render multi-byte characters correctly in streaming', async () => {
+    const html = await renderViaHTTP(next.url, '/multi-byte')
+    expect(html).toContain('マルチバイト'.repeat(28))
+  })
 })