chore: update got to use built-in caching

packages/config/package.json

@@ -76,7 +76,7 @@
     "winston": "^3.8.2"
   },
   "devDependencies": {
-    "@esm2cjs/got": "^12.4.1",
+    "@esm2cjs/got": "^12.5.3",
     "@microsoft/api-extractor": "*",
     "@types/fs-extra": "^9.0.13",
     "@types/jest": "^29.0.2",

packages/zwave-js/package.json

@@ -104,7 +104,7 @@
   "dependencies": {
     "@alcalzone/jsonl-db": "^2.5.3",
     "@alcalzone/pak": "^0.8.1",
-    "@esm2cjs/got": "^12.4.1",
+    "@esm2cjs/got": "^12.5.3",
     "@esm2cjs/p-queue": "^7.3.0",
     "@sentry/integrations": "^7.12.1",
     "@sentry/node": "^7.12.1",

packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts

@@ -13,109 +13,18 @@ import { formatId } from "@zwave-js/shared";
 import crypto from "crypto";
 import type { FirmwareUpdateFileInfo, FirmwareUpdateInfo } from "./_Types";
 
-const serviceURL = "https://firmware.zwave-js.io";
+function serviceURL(): string {
+	return process.env.ZWAVEJS_FW_SERVICE_URL || "https://firmware.zwave-js.io";
+}
+
 const DOWNLOAD_TIMEOUT = 60000;
 // const MAX_FIRMWARE_SIZE = 10 * 1024 * 1024; // 10MB should be enough for any conceivable Z-Wave chip
 
-const MAX_CACHE_SECONDS = 60 * 60 * 24; // Cache for a day at max
-const CLEAN_CACHE_INTERVAL_MS = 60 * 60 * 1000; // Remove stale entries from the cache every hour
-
-const requestCache = new Map<string, CachedRequest<unknown>>();
-interface CachedRequest<T> {
-	response: T;
-	staleDate: number;
-}
+const requestCache = new Map();
 
 // Queue requests to the firmware update service. Only allow few parallel requests so we can make some use of the cache.
 const requestQueue = new PQueue({ concurrency: 2 });
 
-let cleanCacheTimeout: NodeJS.Timeout | undefined;
-function cleanCache() {
-	if (cleanCacheTimeout) {
-		clearTimeout(cleanCacheTimeout);
-		cleanCacheTimeout = undefined;
-	}
-
-	const now = Date.now();
-	for (const [key, cached] of requestCache) {
-		if (cached.staleDate < now) {
-			requestCache.delete(key);
-		}
-	}
-
-	if (requestCache.size > 0) {
-		cleanCacheTimeout = setTimeout(
-			cleanCache,
-			CLEAN_CACHE_INTERVAL_MS,
-		).unref();
-	}
-}
-
-async function cachedGot<T>(config: OptionsOfTextResponseBody): Promise<T> {
-	// Replaces got's built-in cache functionality because it depends on an outdated version of
-	// cacheable-request (<8.3.1), which does not distinguish between POSTs with different bodies
-
-	const hash = crypto
-		.createHash("sha256")
-		.update(JSON.stringify(config.json))
-		.digest("hex");
-	const cacheKey = `${config.method}:${config.url!.toString()}:${hash}`;
-
-	// Return cached requests if they are not stale yet
-	if (requestCache.has(cacheKey)) {
-		const cached = requestCache.get(cacheKey)!;
-		if (cached.staleDate > Date.now()) {
-			return cached.response as T;
-		}
-	}
-
-	const response = await got(config);
-	const responseJson = JSON.parse(response.body) as T;
-
-	// Check if we can cache the response
-	if (response.statusCode === 200 && response.headers["cache-control"]) {
-		const cacheControl = response.headers["cache-control"]!;
-
-		let maxAge: number | undefined;
-		const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
-		if (maxAgeMatch) {
-			maxAge = Math.max(0, parseInt(maxAgeMatch[1], 10));
-		}
-
-		if (maxAge) {
-			let currentAge: number;
-			if (response.headers.age) {
-				currentAge = parseInt(response.headers.age, 10);
-			} else if (response.headers.date) {
-				currentAge =
-					(Date.now() - Date.parse(response.headers.date)) / 1000;
-			} else {
-				currentAge = 0;
-			}
-			currentAge = Math.max(0, currentAge);
-
-			if (maxAge > currentAge) {
-				requestCache.set(cacheKey, {
-					response: responseJson,
-					staleDate:
-						Date.now() +
-						Math.min(MAX_CACHE_SECONDS, maxAge - currentAge) * 1000,
-				});
-			}
-		}
-	}
-
-	// Regularly clean the cache
-	if (!cleanCacheTimeout) {
-		cleanCacheTimeout = setTimeout(
-			cleanCache,
-			CLEAN_CACHE_INTERVAL_MS,
-		).unref();
-	}
-
-	return responseJson;
-}
-
 export interface GetAvailableFirmwareUpdateOptions {
 	userAgent: string;
 	apiKey?: string;
@@ -162,6 +71,7 @@ export function getAvailableFirmwareUpdates(
 ): Promise<FirmwareUpdateInfo[]> {
 	const headers: Headers = {
 		"User-Agent": options.userAgent,
+		"Content-Type": "application/json",
 	};
 	if (options.apiKey) {
 		headers["X-API-Key"] = options.apiKey;
@@ -180,19 +90,20 @@ export function getAvailableFirmwareUpdates(
 
 	const config: OptionsOfTextResponseBody = {
 		method: "POST",
-		url: `${serviceURL}/api/${
+		url: `${serviceURL()}/api/${
 			options.includePrereleases ? "v3" : "v1"
 		}/updates`,
 		json: body,
-		// TODO: Re-enable this in favor of cachedGot when fixed
-		// cache: requestCache,
-		// cacheOptions: {
-		// 	shared: false,
-		// },
+		cache: requestCache,
+		cacheOptions: {
+			shared: false,
+		},
 		headers,
 	};
 
-	return requestQueue.add(() => cachedGot(config));
+	return requestQueue.add(() => {
+		return got(config).json();
+	});
 }
 
 export async function downloadFirmwareUpdate(

test/firmware-update.ts

@@ -0,0 +1,40 @@
+import assert from "assert";
+import { getAvailableFirmwareUpdates } from "../packages/zwave-js/src/lib/controller/FirmwareUpdateService";
+
+async function main() {
+	const get1 = () =>
+		getAvailableFirmwareUpdates(
+			{
+				manufacturerId: 0x0063,
+				productType: 0x4952,
+				productId: 0x3138,
+				firmwareVersion: "1.0",
+			},
+			{
+				userAgent: "TEST",
+			},
+		);
+
+	const get2 = () =>
+		getAvailableFirmwareUpdates(
+			{
+				manufacturerId: 0x0063,
+				productType: 0x4952,
+				productId: 0x3131,
+				firmwareVersion: "1.0",
+			},
+			{
+				userAgent: "TEST",
+			},
+		);
+
+	const upd1 = await get1();
+	const upd2 = await get2();
+
+	console.dir(upd1);
+	console.dir(upd2);
+
+	assert.notDeepStrictEqual(upd1, upd2);
+}
+
+void main();

test/run.ts

@@ -1,8 +1,11 @@
+import { wait as _wait } from "alcalzone-shared/async";
 import os from "os";
 import path from "path";
 import "reflect-metadata";
 import { Driver } from "zwave-js";
 
+const wait = _wait;
+
 process.on("unhandledRejection", (_r) => {
 	debugger;
 });