miniflare: compress responses more like Cloudflare FL (#5798)

* chore: fix whitespace (use tabs within tempate literal strings)

* fix: compress responses more like Cloudflare FL

* add changeset

* fix import order

* nits

* chore: refactor undici imports to use namespace

* fix: remove Content-Encoding header in dispatchFetch
since undici has already decompressed the response body

* add test assertions + explanatory comments

Author: RamIdeas

.changeset/shy-plants-push.md

@@ -0,0 +1,5 @@
+---
+"miniflare": patch
+---
+
+fix: update miniflare's response compression to act more like Cloudflare platform

packages/miniflare/src/http/fetch.ts

@@ -1,21 +1,21 @@
 import http from "http";
 import { IncomingRequestCfProperties } from "@cloudflare/workers-types/experimental";
-import { Dispatcher, Headers, fetch as baseFetch } from "undici";
+import * as undici from "undici";
 import NodeWebSocket from "ws";
 import { CoreHeaders, DeferredPromise } from "../workers";
 import { Request, RequestInfo, RequestInit } from "./request";
 import { Response } from "./response";
 import { WebSocketPair, coupleWebSocket } from "./websocket";
 
 const ignored = ["transfer-encoding", "connection", "keep-alive", "expect"];
-function headersFromIncomingRequest(req: http.IncomingMessage): Headers {
+function headersFromIncomingRequest(req: http.IncomingMessage): undici.Headers {
 	const entries = Object.entries(req.headers).filter(
 		(pair): pair is [string, string | string[]] => {
 			const [name, value] = pair;
 			return !ignored.includes(name) && value !== undefined;
 		}
 	);
-	return new Headers(Object.fromEntries(entries));
+	return new undici.Headers(Object.fromEntries(entries));
 }
 
 export async function fetch(
@@ -87,7 +87,7 @@ export async function fetch(
 		return responsePromise;
 	}
 
-	const response = await baseFetch(request, {
+	const response = await undici.fetch(request, {
 		dispatcher: requestInit?.dispatcher,
 	});
 	return new Response(response.body, response);
@@ -110,7 +110,7 @@ function addHeader(/* mut */ headers: AnyHeaders, key: string, value: string) {
  * `workerd` server is listening on. Handles cases where `fetch()` redirects to
  * same origin and different external origins.
  */
-export class DispatchFetchDispatcher extends Dispatcher {
+export class DispatchFetchDispatcher extends undici.Dispatcher {
 	private readonly cfBlobJson?: string;
 
 	/**
@@ -124,8 +124,8 @@ export class DispatchFetchDispatcher extends Dispatcher {
 	 * @param cfBlob							`request.cf` blob override for runtime requests
 	 */
 	constructor(
-		private readonly globalDispatcher: Dispatcher,
-		private readonly runtimeDispatcher: Dispatcher,
+		private readonly globalDispatcher: undici.Dispatcher,
+		private readonly runtimeDispatcher: undici.Dispatcher,
 		private readonly actualRuntimeOrigin: string,
 		private readonly userRuntimeOrigin: string,
 		cfBlob?: IncomingRequestCfProperties
@@ -149,8 +149,8 @@ export class DispatchFetchDispatcher extends Dispatcher {
 	}
 
 	dispatch(
-		/* mut */ options: Dispatcher.DispatchOptions,
-		handler: Dispatcher.DispatchHandlers
+		/* mut */ options: undici.Dispatcher.DispatchOptions,
+		handler: undici.Dispatcher.DispatchHandlers
 	): boolean {
 		let origin = String(options.origin);
 		// The first request in a redirect chain will always match the user origin

packages/miniflare/src/index.ts

@@ -103,6 +103,7 @@ import {
 	parseWithRootPath,
 	stripAnsi,
 } from "./shared";
+import { isCompressedByCloudflareFL } from "./shared/mime-types";
 import {
 	CoreBindings,
 	CoreHeaders,
@@ -546,9 +547,11 @@ const restrictedWebSocketUpgradeHeaders = [
 	"sec-websocket-accept",
 ];
 
-export function _transformsForContentEncoding(encoding?: string): Transform[] {
+export function _transformsForContentEncodingAndContentType(encoding: string | undefined, type: string | undefined | null): Transform[] {
 	const encoders: Transform[] = [];
 	if (!encoding) return encoders;
+	// if cloudflare's FL does not compress this mime-type, then don't compress locally either
+	if (!isCompressedByCloudflareFL(type)) return encoders;
 
 	// Reverse of https://github.com/nodejs/undici/blob/48d9578f431cbbd6e74f77455ba92184f57096cf/lib/fetch/index.js#L1660
 	const codings = encoding
@@ -587,7 +590,8 @@ async function writeResponse(response: Response, res: http.ServerResponse) {
 	// If a `Content-Encoding` header is set, we'll need to encode the body
 	// (likely only set by custom service bindings)
 	const encoding = headers["content-encoding"]?.toString();
-	const encoders = _transformsForContentEncoding(encoding);
+	const type = headers["content-type"]?.toString();
+	const encoders = _transformsForContentEncodingAndContentType(encoding, type);
 	if (encoders.length > 0) {
 		// `Content-Length` if set, will be wrong as it's for the decoded length
 		delete headers["content-length"];
@@ -1586,6 +1590,16 @@ export class Miniflare {
 			throw reviveError(this.#workerSrcOpts, caught);
 		}
 
+		// At this point, undici.fetch (used inside fetch, above)
+		// has decompressed the response body but retained the Content-Encoding header.
+		// This can cause problems for client implementations which rely
+		// on the Content-Encoding header rather than trying to infer it from the body.
+		// Technically, at this point, this a malformed response so let's remove the header
+		// Retain it as MF-Content-Encoding so we can tell the body was actually compressed.
+		const contentEncoding = response.headers.get('Content-Encoding');
+		if (contentEncoding) response.headers.set('MF-Content-Encoding', contentEncoding);
+		response.headers.delete('Content-Encoding');
+
 		if (
 			process.env.MINIFLARE_ASSERT_BODIES_CONSUMED === "true" &&
 			response.body !== null

packages/miniflare/src/shared/mime-types.ts

@@ -0,0 +1,59 @@
+
+export const compressedByCloudflareFL = new Set([
+    // list copied from https://developers.cloudflare.com/speed/optimization/content/brotli/content-compression/#:~:text=If%20supported%20by%20visitors%E2%80%99%20web%20browsers%2C%20Cloudflare%20will%20return%20Gzip%20or%20Brotli%2Dencoded%20responses%20for%20the%20following%20content%20types%3A
+    "text/html",
+    "text/richtext",
+    "text/plain",
+    "text/css",
+    "text/x-script",
+    "text/x-component",
+    "text/x-java-source",
+    "text/x-markdown",
+    "application/javascript",
+    "application/x-javascript",
+    "text/javascript",
+    "text/js",
+    "image/x-icon",
+    "image/vnd.microsoft.icon",
+    "application/x-perl",
+    "application/x-httpd-cgi",
+    "text/xml",
+    "application/xml",
+    "application/rss+xml",
+    "application/vnd.api+json",
+    "application/x-protobuf",
+    "application/json",
+    "multipart/bag",
+    "multipart/mixed",
+    "application/xhtml+xml",
+    "font/ttf",
+    "font/otf",
+    "font/x-woff",
+    "image/svg+xml",
+    "application/vnd.ms-fontobject",
+    "application/ttf",
+    "application/x-ttf",
+    "application/otf",
+    "application/x-otf",
+    "application/truetype",
+    "application/opentype",
+    "application/x-opentype",
+    "application/font-woff",
+    "application/eot",
+    "application/font",
+    "application/font-sfnt",
+    "application/wasm",
+    "application/javascript-binast",
+    "application/manifest+json",
+    "application/ld+json",
+    "application/graphql+json",
+    "application/geo+json",
+]);
+
+export function isCompressedByCloudflareFL(contentTypeHeader: string | undefined | null) {
+    if (!contentTypeHeader) return true; // Content-Type inferred as text/plain
+
+    const [contentType] = contentTypeHeader.split(';');
+
+    return compressedByCloudflareFL.has(contentType);
+}

packages/miniflare/src/workers/core/entry.worker.ts

@@ -9,6 +9,7 @@ import {
 	yellow,
 } from "kleur/colors";
 import { HttpError, LogLevel, SharedHeaders } from "miniflare:shared";
+import { isCompressedByCloudflareFL } from '../../shared/mime-types';
 import { CoreBindings, CoreHeaders } from "./constants";
 import { STATUS_CODES } from "./http";
 import { WorkerRoute, matchRoutes } from "./routing";
@@ -223,6 +224,12 @@ function ensureAcceptableEncoding(
 	if (encodings.length === 0) return response;
 
 	const contentEncoding = response.headers.get("Content-Encoding");
+	const contentType = response.headers.get("Content-Type");
+
+	// if cloudflare's FL does not compress this mime-type, then don't compress locally either
+	if (!isCompressedByCloudflareFL(contentType)) {
+		return response;
+	}
 
 	// If `Content-Encoding` is defined, but unknown, return the response as is
 	if (