From 02261f27d9d3a6b83087d12b8e653d0039176a83 Mon Sep 17 00:00:00 2001 From: Max Rozen <3822106+rozenmd@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:43:21 +0200 Subject: [PATCH] feat: teach unstable_dev about the internet (remote mode) (#1967) feat: teach unstable_dev about the internet --- .changeset/wet-pens-watch.md | 17 +++ .github/workflows/pullrequests.yml | 3 + .../tests/unstableDev.test.ts | 49 +++++- packages/wrangler/src/api/dev.ts | 2 +- packages/wrangler/src/dev.tsx | 2 +- packages/wrangler/src/dev/remote.tsx | 140 ++++++++++++++++-- packages/wrangler/src/dev/start-server.ts | 101 ++++++++----- packages/wrangler/src/proxy.ts | 74 +++++++++ 8 files changed, 342 insertions(+), 46 deletions(-) create mode 100644 .changeset/wet-pens-watch.md diff --git a/.changeset/wet-pens-watch.md b/.changeset/wet-pens-watch.md new file mode 100644 index 00000000000..c40a010b5a2 --- /dev/null +++ b/.changeset/wet-pens-watch.md @@ -0,0 +1,17 @@ +--- +"wrangler": patch +--- + +feat: implement remote mode for unstable_dev + +With this change, `unstable_dev` can now perform end-to-end (e2e) tests against your workers as you dev. + +Note that to use this feature in CI, you'll need to configure `CLOUDFLARE_API_TOKEN` as an environment variable in your CI, and potentially add `CLOUDFLARE_ACCOUNT_ID` as an environment variable in your CI, or `account_id` in your `wrangler.toml`. + +Usage: + +```js +await unstable_dev("src/index.ts", { + local: false, +}); +``` diff --git a/.github/workflows/pullrequests.yml b/.github/workflows/pullrequests.yml index c2f36d9ddb8..5b396be9e7b 100644 --- a/.github/workflows/pullrequests.yml +++ b/.github/workflows/pullrequests.yml @@ -60,6 +60,9 @@ jobs: - name: Run tests & collect coverage run: npm run test:ci + env: + TMP_CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + TMP_CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - name: Report Code Coverage uses: codecov/codecov-action@v3 diff --git a/fixtures/local-mode-tests/tests/unstableDev.test.ts b/fixtures/local-mode-tests/tests/unstableDev.test.ts index 0868ae1131b..5ca812cf233 100644 --- a/fixtures/local-mode-tests/tests/unstableDev.test.ts +++ b/fixtures/local-mode-tests/tests/unstableDev.test.ts @@ -3,7 +3,7 @@ import { unstable_dev } from "wrangler"; // TODO: add test for `experimentalLocal: true` once issue with dynamic // `import()` and `npx-import` resolved: // https://github.com/cloudflare/wrangler2/pull/1940#issuecomment-1261166695 -describe("worker", () => { +describe("worker in local mode", () => { let worker: { fetch: ( input?: RequestInfo, @@ -19,6 +19,7 @@ describe("worker", () => { { ip: "127.0.0.1", port: 1337, + local: true, }, { disableExperimentalWarning: true } ); @@ -38,3 +39,49 @@ describe("worker", () => { } }); }); + +describe("worker in remote mode", () => { + let worker: { + fetch: ( + input?: RequestInfo, + init?: RequestInit + ) => Promise; + stop: () => Promise; + }; + + beforeAll(async () => { + if (process.env.TMP_CLOUDFLARE_API_TOKEN) { + process.env.CLOUDFLARE_API_TOKEN = process.env.TMP_CLOUDFLARE_API_TOKEN; + } + + if (process.env.TMP_CLOUDFLARE_ACCOUNT_ID) { + process.env.CLOUDFLARE_ACCOUNT_ID = process.env.TMP_CLOUDFLARE_ACCOUNT_ID; + } + + //since the script is invoked from the directory above, need to specify index.js is in src/ + worker = await unstable_dev( + "src/basicModule.ts", + { + ip: "127.0.0.1", + port: 1337, + local: false, + }, + { disableExperimentalWarning: true } + ); + }); + + afterAll(async () => { + await worker?.stop(); + process.env.CLOUDFLARE_API_TOKEN = undefined; + }); + + it("should invoke the worker and exit", async () => { + const resp = await worker.fetch(); + expect(resp).not.toBe(undefined); + if (resp) { + const text = await resp.text(); + + expect(text).toMatchInlineSnapshot(`"Hello World!"`); + } + }); +}); diff --git a/packages/wrangler/src/api/dev.ts b/packages/wrangler/src/api/dev.ts index 1711c1dc25e..85058359a33 100644 --- a/packages/wrangler/src/api/dev.ts +++ b/packages/wrangler/src/api/dev.ts @@ -102,8 +102,8 @@ export async function unstable_dev( _: [], $0: "", port: options?.port ?? 0, - ...options, local: true, + ...options, onReady: (address, port) => { readyPort = port; readyAddress = address; diff --git a/packages/wrangler/src/dev.tsx b/packages/wrangler/src/dev.tsx index d6ead4b1691..c647d68800c 100644 --- a/packages/wrangler/src/dev.tsx +++ b/packages/wrangler/src/dev.tsx @@ -577,7 +577,7 @@ export async function startApiDev(args: StartDevOptions) { showInteractiveDevSession: args.showInteractiveDevSession, forceLocal: args.forceLocal, enablePagesAssetsServiceBinding: args.enablePagesAssetsServiceBinding, - local: true, + local: args.local ?? true, firstPartyWorker: configParam.first_party_worker, sendMetrics: configParam.send_metrics, testScheduled: args.testScheduled, diff --git a/packages/wrangler/src/dev/remote.tsx b/packages/wrangler/src/dev/remote.tsx index 607602b3c9c..1e09360dde0 100644 --- a/packages/wrangler/src/dev/remote.tsx +++ b/packages/wrangler/src/dev/remote.tsx @@ -9,7 +9,7 @@ import { } from "../create-worker-preview"; import useInspector from "../inspect"; import { logger } from "../logger"; -import { usePreviewServer } from "../proxy"; +import { startPreviewServer, usePreviewServer } from "../proxy"; import { syncAssets } from "../sites"; import { ChooseAccount, @@ -150,7 +150,7 @@ export function Remote(props: RemoteProps) { ) : null; } -export function useWorker(props: { +interface RemoteWorkerProps { name: string | undefined; bundle: EsbuildBundle | undefined; format: CfScriptFormat | undefined; @@ -170,7 +170,11 @@ export function useWorker(props: { onReady: ((ip: string, port: number) => void) | undefined; sendMetrics: boolean | undefined; port: number; -}): CfPreviewToken | undefined { +} + +export function useWorker( + props: RemoteWorkerProps +): CfPreviewToken | undefined { const [session, setSession] = useState(); const [token, setToken] = useState(); const [restartCounter, setRestartCounter] = useState(0); @@ -263,24 +267,20 @@ export function useWorker(props: { usageModel: props.usageModel, }); - const workerAccount: CfAccount = { + const { workerAccount, workerContext } = getWorkerAccountAndContext({ accountId: props.accountId, - apiToken: requireApiToken(), - }; - - const workerCtx: CfWorkerContext = { env: props.env, legacyEnv: props.legacyEnv, zone: props.zone, host: props.host, routes: props.routes, sendMetrics: props.sendMetrics, - }; + }); const workerPreviewToken = await createWorkerPreview( init, workerAccount, - workerCtx, + workerContext, session, abortController.signal ); @@ -367,6 +367,126 @@ export function useWorker(props: { return token; } +export async function startRemoteServer(props: RemoteProps) { + let accountId = props.accountId; + if (accountId === undefined) { + const accountChoices = await getAccountChoices(); + if (accountChoices.length === 1) { + saveAccountToCache({ + id: accountChoices[0].id, + name: accountChoices[0].name, + }); + accountId = accountChoices[0].id; + } else { + throw logger.error( + "In a non-interactive environment, it is mandatory to specify an account ID, either by assigning its value to CLOUDFLARE_ACCOUNT_ID, or as `account_id` in your `wrangler.toml` file." + ); + } + } + + const previewToken = await getRemotePreviewToken({ + ...props, + accountId: accountId, + }); + + if (previewToken === undefined) { + throw logger.error("Failed to get a previewToken"); + } + // start our proxy server + const previewServer = await startPreviewServer({ + previewToken, + assetDirectory: props.isWorkersSite + ? undefined + : props.assetPaths?.assetDirectory, + localProtocol: props.localProtocol, + localPort: props.port, + ip: props.ip, + onReady: props.onReady, + }); + if (!previewServer) { + throw logger.error("Failed to start remote server"); + } + return { stop: previewServer.stop }; +} + +/** + * getRemotePreviewToken is a react-free version of `useWorker`. + * It returns a preview token, which we then use in our proxy server + */ +export async function getRemotePreviewToken(props: RemoteProps) { + //setup the preview session + async function start() { + if (props.accountId === undefined) { + throw logger.error("no accountId provided"); + } + const abortController = new AbortController(); + const { workerAccount, workerContext } = getWorkerAccountAndContext({ + accountId: props.accountId, + env: props.env, + legacyEnv: props.legacyEnv, + zone: props.zone, + host: props.host, + routes: props.routes, + sendMetrics: props.sendMetrics, + }); + const session = await createPreviewSession( + workerAccount, + workerContext, + abortController.signal + ); + //use the session to upload the worker, and create a preview + + if (session === undefined) { + throw logger.error("Failed to start a session"); + } + if (!props.bundle || !props.format) return; + + const init = await createRemoteWorkerInit({ + bundle: props.bundle, + modules: props.bundle ? props.bundle.modules : [], + accountId: props.accountId, + name: props.name, + legacyEnv: props.legacyEnv, + env: props.env, + isWorkersSite: props.isWorkersSite, + assetPaths: props.assetPaths, + format: props.format, + bindings: props.bindings, + compatibilityDate: props.compatibilityDate, + compatibilityFlags: props.compatibilityFlags, + usageModel: props.usageModel, + }); + const workerPreviewToken = await createWorkerPreview( + init, + workerAccount, + workerContext, + session, + abortController.signal + ); + return workerPreviewToken; + } + return start().catch((err) => { + if ((err as { code?: string })?.code !== "ABORT_ERR") { + // instead of logging the raw API error to the user, + // give them friendly instructions + // for error 10063 (workers.dev subdomain required) + if (err?.code === 10063) { + const errorMessage = + "Error: You need to register a workers.dev subdomain before running the dev command in remote mode"; + const solutionMessage = + "You can either enable local mode by pressing l, or register a workers.dev subdomain here:"; + const onboardingLink = `https://dash.cloudflare.com/${props.accountId}/workers/onboarding`; + logger.error(`${errorMessage}\n${solutionMessage}\n${onboardingLink}`); + } else if (err?.code === 10049) { + // code 10049 happens when the preview token expires + logger.log("Preview token expired, restart server to fetch a new one"); + } else { + logger.error("Error on remote worker:", err); + } + } + }); +} + async function createRemoteWorkerInit(props: { bundle: EsbuildBundle; modules: CfModule[]; diff --git a/packages/wrangler/src/dev/start-server.ts b/packages/wrangler/src/dev/start-server.ts index f3ce3f859d0..3ac259669d5 100644 --- a/packages/wrangler/src/dev/start-server.ts +++ b/packages/wrangler/src/dev/start-server.ts @@ -21,6 +21,7 @@ import { setupMiniflareOptions, setupNodeOptions, } from "./local"; +import { startRemoteServer } from "./remote"; import { validateDevProps } from "./validate-dev-props"; import type { Config } from "../config"; @@ -97,40 +98,74 @@ export async function startDevServer( experimentalLocalStubCache: props.experimentalLocal, }); - //run local now - const { stop, inspectorUrl } = await startLocalServer({ - name: props.name, - bundle: bundle, - format: props.entry.format, - compatibilityDate: props.compatibilityDate, - compatibilityFlags: props.compatibilityFlags, - bindings: props.bindings, - assetPaths: props.assetPaths, - port: props.port, - ip: props.ip, - rules: props.rules, - inspectorPort: props.inspectorPort, - localPersistencePath: props.localPersistencePath, - liveReload: props.liveReload, - crons: props.crons, - localProtocol: props.localProtocol, - localUpstream: props.localUpstream, - logPrefix: props.logPrefix, - inspect: props.inspect, - onReady: props.onReady, - enablePagesAssetsServiceBinding: props.enablePagesAssetsServiceBinding, - usageModel: props.usageModel, - workerDefinitions, - experimentalLocal: props.experimentalLocal, - }); + if (props.local) { + const { stop, inspectorUrl } = await startLocalServer({ + name: props.name, + bundle: bundle, + format: props.entry.format, + compatibilityDate: props.compatibilityDate, + compatibilityFlags: props.compatibilityFlags, + bindings: props.bindings, + assetPaths: props.assetPaths, + port: props.port, + ip: props.ip, + rules: props.rules, + inspectorPort: props.inspectorPort, + localPersistencePath: props.localPersistencePath, + liveReload: props.liveReload, + crons: props.crons, + localProtocol: props.localProtocol, + localUpstream: props.localUpstream, + logPrefix: props.logPrefix, + inspect: props.inspect, + onReady: props.onReady, + enablePagesAssetsServiceBinding: props.enablePagesAssetsServiceBinding, + usageModel: props.usageModel, + workerDefinitions, + experimentalLocal: props.experimentalLocal, + }); - return { - stop: async () => { - stop(); - await stopWorkerRegistry(); - }, - inspectorUrl, - }; + return { + stop: async () => { + stop(); + await stopWorkerRegistry(); + }, + inspectorUrl, + }; + } else { + const { stop } = await startRemoteServer({ + name: props.name, + bundle: bundle, + format: props.entry.format, + accountId: props.accountId, + bindings: props.bindings, + assetPaths: props.assetPaths, + isWorkersSite: props.isWorkersSite, + port: props.port, + ip: props.ip, + localProtocol: props.localProtocol, + inspectorPort: props.inspectorPort, + inspect: props.inspect, + compatibilityDate: props.compatibilityDate, + compatibilityFlags: props.compatibilityFlags, + usageModel: props.usageModel, + env: props.env, + legacyEnv: props.legacyEnv, + zone: props.zone, + host: props.host, + routes: props.routes, + onReady: props.onReady, + sourceMapPath: bundle?.sourceMapPath, + sendMetrics: props.sendMetrics, + }); + return { + stop: async () => { + stop(); + await stopWorkerRegistry(); + }, + // TODO: inspectorUrl, + }; + } } catch (err) { logger.error(err); } diff --git a/packages/wrangler/src/proxy.ts b/packages/wrangler/src/proxy.ts index 309ca243988..96e648f58d7 100644 --- a/packages/wrangler/src/proxy.ts +++ b/packages/wrangler/src/proxy.ts @@ -90,6 +90,80 @@ type PreviewProxy = { terminator: HttpTerminator; }; +export async function startPreviewServer({ + previewToken, + assetDirectory, + localProtocol, + localPort: port, + ip, + onReady, +}: { + previewToken: CfPreviewToken; + assetDirectory: string | undefined; + localProtocol: "https" | "http"; + localPort: number; + ip: string; + onReady: ((readyIp: string, readyPort: number) => void) | undefined; +}) { + try { + const abortController = new AbortController(); + + const server = await createProxyServer(localProtocol); + const proxy = { + server, + terminator: createHttpTerminator({ + server, + gracefulTerminationTimeout: 0, + }), + }; + + // We have a token. Let's proxy requests to the preview end point. + const streamBufferRef = { current: [] }; + const requestResponseBufferRef = { current: [] }; + const cleanupListeners = configureProxyServer({ + proxy, + previewToken, + streamBufferRef, + requestResponseBufferRef, + retryServerSetup: () => {}, // no-op outside of React + assetDirectory, + localProtocol, + port, + }); + + await waitForPortToBeAvailable(port, { + retryPeriod: 200, + timeout: 2000, + abortSignal: abortController.signal, + }); + + proxy.server.on("listening", () => { + const address = proxy.server.address(); + const usedPort = + address && typeof address === "object" ? address.port : port; + logger.log(`⬣ Listening at ${localProtocol}://${ip}:${usedPort}`); + const accessibleHosts = ip !== "0.0.0.0" ? [ip] : getAccessibleHosts(); + for (const accessibleHost of accessibleHosts) { + logger.log(`- ${localProtocol}://${accessibleHost}:${usedPort}`); + } + onReady?.(ip, usedPort); + }); + + proxy.server.listen(port, ip); + return { + stop: () => { + abortController.abort(); + cleanupListeners?.forEach((cleanup) => cleanup()); + }, + }; + } catch (err) { + if ((err as { code: string }).code !== "ABORT_ERR") { + logger.error(`Failed to start server: ${err}`); + } + logger.error("Failed to create proxy server:", err); + } +} + export function usePreviewServer({ previewToken, assetDirectory,