From 52abeb2d9bd5c56c07b8986b736a9f8c86e61948 Mon Sep 17 00:00:00 2001 From: Dominic Griesel Date: Tue, 20 Sep 2022 11:13:17 +0200 Subject: [PATCH 1/2] chore: update got to use built-in caching --- packages/config/package.json | 2 +- packages/zwave-js/package.json | 2 +- .../lib/controller/FirmwareUpdateService.ts | 113 ++----------- test/firmware-update.ts | 40 +++++ yarn.lock | 160 ++++++------------ 5 files changed, 101 insertions(+), 216 deletions(-) create mode 100644 test/firmware-update.ts diff --git a/packages/config/package.json b/packages/config/package.json index 0e63fc3e5c9..badf8a566f6 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -76,7 +76,7 @@ "winston": "^3.8.2" }, "devDependencies": { - "@esm2cjs/got": "^12.4.1", + "@esm2cjs/got": "^12.5.0", "@microsoft/api-extractor": "*", "@types/fs-extra": "^9.0.13", "@types/jest": "^29.0.2", diff --git a/packages/zwave-js/package.json b/packages/zwave-js/package.json index eb5aa12c780..66d56d891d1 100644 --- a/packages/zwave-js/package.json +++ b/packages/zwave-js/package.json @@ -104,7 +104,7 @@ "dependencies": { "@alcalzone/jsonl-db": "^2.5.3", "@alcalzone/pak": "^0.8.1", - "@esm2cjs/got": "^12.4.1", + "@esm2cjs/got": "^12.5.0", "@esm2cjs/p-queue": "^7.3.0", "@sentry/integrations": "^7.12.1", "@sentry/node": "^7.12.1", diff --git a/packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts b/packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts index 46912ac2632..a1faab2f70c 100644 --- a/packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts +++ b/packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts @@ -12,109 +12,16 @@ import { formatId } from "@zwave-js/shared"; import crypto from "crypto"; import type { FirmwareUpdateFileInfo, FirmwareUpdateInfo } from "./_Types"; -const serviceURL = "https://firmware.zwave-js.io"; +const serviceURL = "http://127.0.0.1:8787"; +// const serviceURL = "https://firmware.zwave-js.io"; const DOWNLOAD_TIMEOUT = 60000; // const MAX_FIRMWARE_SIZE = 10 * 1024 * 1024; // 10MB should be enough for any conceivable Z-Wave chip -const MAX_CACHE_SECONDS = 60 * 60 * 24; // Cache for a day at max -const CLEAN_CACHE_INTERVAL_MS = 60 * 60 * 1000; // Remove stale entries from the cache every hour - -const requestCache = new Map>(); -interface CachedRequest { - response: T; - staleDate: number; -} +const requestCache = new Map(); // Queue requests to the firmware update service. Only allow few parallel requests so we can make some use of the cache. const requestQueue = new PQueue({ concurrency: 2 }); -let cleanCacheTimeout: NodeJS.Timeout | undefined; -function cleanCache() { - if (cleanCacheTimeout) { - clearTimeout(cleanCacheTimeout); - cleanCacheTimeout = undefined; - } - - const now = Date.now(); - for (const [key, cached] of requestCache) { - if (cached.staleDate < now) { - requestCache.delete(key); - } - } - - if (requestCache.size > 0) { - cleanCacheTimeout = setTimeout( - cleanCache, - CLEAN_CACHE_INTERVAL_MS, - ).unref(); - } -} - -async function cachedGot(config: OptionsOfTextResponseBody): Promise { - // Replaces got's built-in cache functionality because it depends on an outdated version of - // cacheable-request (<8.3.1), which does not distinguish between POSTs with different bodies - - const hash = crypto - .createHash("sha256") - .update(JSON.stringify(config.json)) - .digest("hex"); - const cacheKey = `${config.method}:${config.url!.toString()}:${hash}`; - - // Return cached requests if they are not stale yet - if (requestCache.has(cacheKey)) { - const cached = requestCache.get(cacheKey)!; - if (cached.staleDate > Date.now()) { - return cached.response as T; - } - } - - const response = await got(config); - const responseJson = JSON.parse(response.body) as T; - - // Check if we can cache the response - if (response.statusCode === 200 && response.headers["cache-control"]) { - const cacheControl = response.headers["cache-control"]!; - - let maxAge: number | undefined; - const maxAgeMatch = cacheControl.match(/max-age=(\d+)/); - if (maxAgeMatch) { - maxAge = Math.max(0, parseInt(maxAgeMatch[1], 10)); - } - - if (maxAge) { - let currentAge: number; - if (response.headers.age) { - currentAge = parseInt(response.headers.age, 10); - } else if (response.headers.date) { - currentAge = - (Date.now() - Date.parse(response.headers.date)) / 1000; - } else { - currentAge = 0; - } - currentAge = Math.max(0, currentAge); - - if (maxAge > currentAge) { - requestCache.set(cacheKey, { - response: responseJson, - staleDate: - Date.now() + - Math.min(MAX_CACHE_SECONDS, maxAge - currentAge) * 1000, - }); - } - } - } - - // Regularly clean the cache - if (!cleanCacheTimeout) { - cleanCacheTimeout = setTimeout( - cleanCache, - CLEAN_CACHE_INTERVAL_MS, - ).unref(); - } - - return responseJson; -} - export interface GetAvailableFirmwareUpdateOptions { userAgent: string; apiKey?: string; @@ -130,6 +37,7 @@ export function getAvailableFirmwareUpdates( ): Promise { const headers: Headers = { "User-Agent": options.userAgent, + "Content-Type": "application/json", }; if (options.apiKey) { headers["X-API-Key"] = options.apiKey; @@ -144,15 +52,16 @@ export function getAvailableFirmwareUpdates( productId: formatId(deviceId.productId), firmwareVersion: deviceId.firmwareVersion, }, - // TODO: Re-enable this in favor of cachedGot when fixed - // cache: requestCache, - // cacheOptions: { - // shared: false, - // }, + cache: requestCache, + cacheOptions: { + shared: false, + }, headers, }; - return requestQueue.add(() => cachedGot(config)); + return requestQueue.add(() => { + return got(config).json(); + }); } export async function downloadFirmwareUpdate( diff --git a/test/firmware-update.ts b/test/firmware-update.ts new file mode 100644 index 00000000000..02d7c2e4773 --- /dev/null +++ b/test/firmware-update.ts @@ -0,0 +1,40 @@ +import assert from "assert"; +import { getAvailableFirmwareUpdates } from "../packages/zwave-js/src/lib/controller/FirmwareUpdateService"; + +async function main() { + const get1 = () => + getAvailableFirmwareUpdates( + { + manufacturerId: 0x0063, + productType: 0x4952, + productId: 0x3138, + firmwareVersion: "1.0", + }, + { + userAgent: "TEST", + }, + ); + + const get2 = () => + getAvailableFirmwareUpdates( + { + manufacturerId: 0x0063, + productType: 0x4952, + productId: 0x3131, + firmwareVersion: "1.0", + }, + { + userAgent: "TEST", + }, + ); + + const upd1 = await get1(); + const upd2 = await get2(); + + console.dir(upd1); + console.dir(upd2); + + assert.notDeepStrictEqual(upd1, upd2); +} + +void main(); diff --git a/yarn.lock b/yarn.lock index 9674e8969b2..6473b64f68f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2916,30 +2916,43 @@ __metadata: languageName: node linkType: hard -"@esm2cjs/form-data-encoder@npm:^2.1.0": +"@esm2cjs/cacheable-request@npm:^10.1.2": + version: 10.1.2 + resolution: "@esm2cjs/cacheable-request@npm:10.1.2" + dependencies: + "@esm2cjs/mimic-response": ^4.0.0 + "@esm2cjs/normalize-url": ^7.1.0 + "@esm2cjs/responselike": ^3.0.0 + get-stream: ^6.0.1 + http-cache-semantics: ^4.1.0 + keyv: ^4.5.0 + checksum: 0ac726528044cbc4407a41fd46b72ab3126e242a040ce589812a774f829c3fa28270cc970f868c1b5a74261c6f0137ca9deaa2201acdcd5d90b4148e381a69ed + languageName: node + linkType: hard + +"@esm2cjs/form-data-encoder@npm:^2.1.2": version: 2.1.2 resolution: "@esm2cjs/form-data-encoder@npm:2.1.2" checksum: d6430e5c58ae219d13fe208958690c24487189af2a0373505ff99c975d167466736317e1e2d87fa677f5a25360d93e32fdf14c180e7a95e41f9bbefc0d85e489 languageName: node linkType: hard -"@esm2cjs/got@npm:^12.4.1": - version: 12.4.1 - resolution: "@esm2cjs/got@npm:12.4.1" +"@esm2cjs/got@npm:^12.5.0": + version: 12.5.0 + resolution: "@esm2cjs/got@npm:12.5.0" dependencies: - "@esm2cjs/form-data-encoder": ^2.1.0 + "@esm2cjs/cacheable-request": ^10.1.2 + "@esm2cjs/form-data-encoder": ^2.1.2 "@esm2cjs/http-timer": ^5.0.1 "@esm2cjs/is": ^5.2.0 "@esm2cjs/lowercase-keys": ^3.0.0 "@esm2cjs/p-cancelable": ^3.0.0 - "@types/cacheable-request": ^6.0.2 cacheable-lookup: ^6.0.4 - cacheable-request: ^7.0.2 decompress-response: ^6.0.0 get-stream: ^6.0.1 http2-wrapper: ^2.1.10 responselike: ^3.0.0 - checksum: f274a5218a6bb3a0f7c8d2ea857d240e189412769c7cab0ea8d7c7017c6a61603ecac5d9ad173f3b025106d9765571d5d23ad7434b54262fdc288ac0abcf6340 + checksum: bb3ab3b07e1dcf53573f8c5d39cd587fff9792aaadd9a6ab1de2099d62f184a2af90224fecf6067eef833ca0794bd97022ff4d48025909b14050b01c86021bb4 languageName: node linkType: hard @@ -2966,6 +2979,20 @@ __metadata: languageName: node linkType: hard +"@esm2cjs/mimic-response@npm:^4.0.0": + version: 4.0.0 + resolution: "@esm2cjs/mimic-response@npm:4.0.0" + checksum: 1c9c50d0231f630263976c885c1a0371ef1d162a6952da26cb9f87d90189ef85ce1d31cd1ba914738af22b4598f7d4312bc018fa27da35d0d0737cbb12312139 + languageName: node + linkType: hard + +"@esm2cjs/normalize-url@npm:^7.1.0": + version: 7.1.0 + resolution: "@esm2cjs/normalize-url@npm:7.1.0" + checksum: f4fd6251b373fa955a571a922dc0e63648c5c272932a1f238dfa1f42365b1b418a1a1d7270efa3cd72cb53e656d15d478115b6abe3acc513eef79272ed700371 + languageName: node + linkType: hard + "@esm2cjs/p-cancelable@npm:^3.0.0": version: 3.0.0 resolution: "@esm2cjs/p-cancelable@npm:3.0.0" @@ -2990,6 +3017,15 @@ __metadata: languageName: node linkType: hard +"@esm2cjs/responselike@npm:^3.0.0": + version: 3.0.0 + resolution: "@esm2cjs/responselike@npm:3.0.0" + dependencies: + "@esm2cjs/lowercase-keys": ^3.0.0 + checksum: c0c15ac29c56dcdc3c2c96c8732534e048a97e2d8c3c0eadd04568e228b2b15d1150f3d320b5df1a6376e4e20276113ba4c1243b1c540fd6d52ee139b6a80ad6 + languageName: node + linkType: hard + "@humanwhocodes/config-array@npm:^0.10.4": version: 0.10.4 resolution: "@humanwhocodes/config-array@npm:0.10.4" @@ -4045,18 +4081,6 @@ __metadata: languageName: node linkType: hard -"@types/cacheable-request@npm:^6.0.2": - version: 6.0.2 - resolution: "@types/cacheable-request@npm:6.0.2" - dependencies: - "@types/http-cache-semantics": "*" - "@types/keyv": "*" - "@types/node": "*" - "@types/responselike": "*" - checksum: 667d25808dbf46fe104d6f029e0281ff56058d50c7c1b9182774b3e38bb9c1124f56e4c367ba54f92dbde2d1cc573f26eb0e9748710b2822bc0fd1e5498859c6 - languageName: node - linkType: hard - "@types/clipboardy@npm:^2.0.1": version: 2.0.1 resolution: "@types/clipboardy@npm:2.0.1" @@ -4091,13 +4115,6 @@ __metadata: languageName: node linkType: hard -"@types/http-cache-semantics@npm:*": - version: 4.0.1 - resolution: "@types/http-cache-semantics@npm:4.0.1" - checksum: 1048aacf627829f0d5f00184e16548205cd9f964bf0841c29b36bc504509230c40bc57c39778703a1c965a6f5b416ae2cbf4c1d4589c889d2838dd9dbfccf6e9 - languageName: node - linkType: hard - "@types/istanbul-lib-coverage@npm:*, @types/istanbul-lib-coverage@npm:^2.0.0, @types/istanbul-lib-coverage@npm:^2.0.1": version: 2.0.3 resolution: "@types/istanbul-lib-coverage@npm:2.0.3" @@ -4154,15 +4171,6 @@ __metadata: languageName: node linkType: hard -"@types/keyv@npm:*": - version: 3.1.4 - resolution: "@types/keyv@npm:3.1.4" - dependencies: - "@types/node": "*" - checksum: e009a2bfb50e90ca9b7c6e8f648f8464067271fd99116f881073fa6fa76dc8d0133181dd65e6614d5fb1220d671d67b0124aef7d97dc02d7e342ab143a47779d - languageName: node - linkType: hard - "@types/minimist@npm:^1.2.0": version: 1.2.1 resolution: "@types/minimist@npm:1.2.1" @@ -4249,15 +4257,6 @@ __metadata: languageName: node linkType: hard -"@types/responselike@npm:*": - version: 1.0.0 - resolution: "@types/responselike@npm:1.0.0" - dependencies: - "@types/node": "*" - checksum: e99fc7cc6265407987b30deda54c1c24bb1478803faf6037557a774b2f034c5b097ffd65847daa87e82a61a250d919f35c3588654b0fdaa816906650f596d1b0 - languageName: node - linkType: hard - "@types/retry@npm:*": version: 0.12.0 resolution: "@types/retry@npm:0.12.0" @@ -4505,7 +4504,7 @@ __metadata: version: 0.0.0-use.local resolution: "@zwave-js/config@workspace:packages/config" dependencies: - "@esm2cjs/got": ^12.4.1 + "@esm2cjs/got": ^12.5.0 "@microsoft/api-extractor": "*" "@types/fs-extra": ^9.0.13 "@types/jest": ^29.0.2 @@ -5448,21 +5447,6 @@ __metadata: languageName: node linkType: hard -"cacheable-request@npm:^7.0.2": - version: 7.0.2 - resolution: "cacheable-request@npm:7.0.2" - dependencies: - clone-response: ^1.0.2 - get-stream: ^5.1.0 - http-cache-semantics: ^4.0.0 - keyv: ^4.0.0 - lowercase-keys: ^2.0.0 - normalize-url: ^6.0.1 - responselike: ^2.0.0 - checksum: 6152813982945a5c9989cb457a6c499f12edcc7ade323d2fbfd759abc860bdbd1306e08096916bb413c3c47e812f8e4c0a0cc1e112c8ce94381a960f115bc77f - languageName: node - linkType: hard - "cachedir@npm:2.2.0": version: 2.2.0 resolution: "cachedir@npm:2.2.0" @@ -5755,15 +5739,6 @@ __metadata: languageName: node linkType: hard -"clone-response@npm:^1.0.2": - version: 1.0.3 - resolution: "clone-response@npm:1.0.3" - dependencies: - mimic-response: ^1.0.0 - checksum: 4e671cac39b11c60aa8ba0a450657194a5d6504df51bca3fac5b3bd0145c4f8e8464898f87c8406b83232e3bc5cca555f51c1f9c8ac023969ebfbf7f6bdabb2e - languageName: node - linkType: hard - "clone@npm:^1.0.2": version: 1.0.4 resolution: "clone@npm:1.0.4" @@ -7495,15 +7470,6 @@ __metadata: languageName: node linkType: hard -"get-stream@npm:^5.1.0": - version: 5.2.0 - resolution: "get-stream@npm:5.2.0" - dependencies: - pump: ^3.0.0 - checksum: 8bc1a23174a06b2b4ce600df38d6c98d2ef6d84e020c1ddad632ad75bac4e092eeb40e4c09e0761c35fc2dbc5e7fff5dab5e763a383582c4a167dd69a905bd12 - languageName: node - linkType: hard - "get-stream@npm:^6.0.0, get-stream@npm:^6.0.1": version: 6.0.1 resolution: "get-stream@npm:6.0.1" @@ -7850,7 +7816,7 @@ __metadata: languageName: node linkType: hard -"http-cache-semantics@npm:^4.0.0, http-cache-semantics@npm:^4.1.0": +"http-cache-semantics@npm:^4.1.0": version: 4.1.0 resolution: "http-cache-semantics@npm:4.1.0" checksum: 974de94a81c5474be07f269f9fd8383e92ebb5a448208223bfb39e172a9dbc26feff250192ecc23b9593b3f92098e010406b0f24bd4d588d631f80214648ed42 @@ -9037,7 +9003,7 @@ __metadata: languageName: node linkType: hard -"keyv@npm:^4.0.0": +"keyv@npm:^4.5.0": version: 4.5.0 resolution: "keyv@npm:4.5.0" dependencies: @@ -9242,13 +9208,6 @@ __metadata: languageName: node linkType: hard -"lowercase-keys@npm:^2.0.0": - version: 2.0.0 - resolution: "lowercase-keys@npm:2.0.0" - checksum: 24d7ebd56ccdf15ff529ca9e08863f3c54b0b9d1edb97a3ae1af34940ae666c01a1e6d200707bce730a8ef76cb57cc10e65f245ecaaf7e6bc8639f2fb460ac23 - languageName: node - linkType: hard - "lowercase-keys@npm:^3.0.0": version: 3.0.0 resolution: "lowercase-keys@npm:3.0.0" @@ -9441,13 +9400,6 @@ __metadata: languageName: node linkType: hard -"mimic-response@npm:^1.0.0": - version: 1.0.1 - resolution: "mimic-response@npm:1.0.1" - checksum: 034c78753b0e622bc03c983663b1cdf66d03861050e0c8606563d149bc2b02d63f62ce4d32be4ab50d0553ae0ffe647fc34d1f5281184c6e1e8cf4d85e8d9823 - languageName: node - linkType: hard - "mimic-response@npm:^3.1.0": version: 3.1.0 resolution: "mimic-response@npm:3.1.0" @@ -9827,13 +9779,6 @@ __metadata: languageName: node linkType: hard -"normalize-url@npm:^6.0.1": - version: 6.1.0 - resolution: "normalize-url@npm:6.1.0" - checksum: 4a4944631173e7d521d6b80e4c85ccaeceb2870f315584fa30121f505a6dfd86439c5e3fdd8cd9e0e291290c41d0c3599f0cb12ab356722ed242584c30348e50 - languageName: node - linkType: hard - "npm-run-path@npm:^2.0.0": version: 2.0.2 resolution: "npm-run-path@npm:2.0.2" @@ -10852,15 +10797,6 @@ __metadata: languageName: node linkType: hard -"responselike@npm:^2.0.0": - version: 2.0.1 - resolution: "responselike@npm:2.0.1" - dependencies: - lowercase-keys: ^2.0.0 - checksum: b122535466e9c97b55e69c7f18e2be0ce3823c5d47ee8de0d9c0b114aa55741c6db8bfbfce3766a94d1272e61bfb1ebf0a15e9310ac5629fbb7446a861b4fd3a - languageName: node - linkType: hard - "responselike@npm:^3.0.0": version: 3.0.0 resolution: "responselike@npm:3.0.0" @@ -12629,7 +12565,7 @@ __metadata: dependencies: "@alcalzone/jsonl-db": ^2.5.3 "@alcalzone/pak": ^0.8.1 - "@esm2cjs/got": ^12.4.1 + "@esm2cjs/got": ^12.5.0 "@esm2cjs/p-queue": ^7.3.0 "@microsoft/api-extractor": "*" "@sentry/integrations": ^7.12.1 From e73c19d239938433a3c51dfbff43b4e41f633fdb Mon Sep 17 00:00:00 2001 From: Dominic Griesel Date: Wed, 4 Jan 2023 11:50:31 +0100 Subject: [PATCH 2/2] feat: make service URL configurable --- .../zwave-js/src/lib/controller/FirmwareUpdateService.ts | 8 +++++--- test/run.ts | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts b/packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts index 35089a613c7..b4d73d23473 100644 --- a/packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts +++ b/packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts @@ -13,8 +13,10 @@ import { formatId } from "@zwave-js/shared"; import crypto from "crypto"; import type { FirmwareUpdateFileInfo, FirmwareUpdateInfo } from "./_Types"; -const serviceURL = "http://127.0.0.1:8787"; -// const serviceURL = "https://firmware.zwave-js.io"; +function serviceURL(): string { + return process.env.ZWAVEJS_FW_SERVICE_URL || "https://firmware.zwave-js.io"; +} + const DOWNLOAD_TIMEOUT = 60000; // const MAX_FIRMWARE_SIZE = 10 * 1024 * 1024; // 10MB should be enough for any conceivable Z-Wave chip @@ -88,7 +90,7 @@ export function getAvailableFirmwareUpdates( const config: OptionsOfTextResponseBody = { method: "POST", - url: `${serviceURL}/api/${ + url: `${serviceURL()}/api/${ options.includePrereleases ? "v3" : "v1" }/updates`, json: body, diff --git a/test/run.ts b/test/run.ts index 8d1dad52ca9..026045bf777 100644 --- a/test/run.ts +++ b/test/run.ts @@ -1,8 +1,11 @@ +import { wait as _wait } from "alcalzone-shared/async"; import os from "os"; import path from "path"; import "reflect-metadata"; import { Driver } from "zwave-js"; +const wait = _wait; + process.on("unhandledRejection", (_r) => { debugger; });