diff --git a/packages/playwright-core/src/server/browserContext.ts b/packages/playwright-core/src/server/browserContext.ts index 7ab1951829e82..25828282dcbde 100644 --- a/packages/playwright-core/src/server/browserContext.ts +++ b/packages/playwright-core/src/server/browserContext.ts @@ -78,11 +78,12 @@ export abstract class BrowserContext extends SdkObject { // Create instrumentation per context. this.instrumentation = createInstrumentation(); + this.fetchRequest = new BrowserContextAPIRequestContext(this); + if (this._options.recordHar) this._harRecorder = new HarRecorder(this, { ...this._options.recordHar, path: path.join(this._browser.options.artifactsDir, `${createGuid()}.har`) }); this.tracing = new Tracing(this); - this.fetchRequest = new BrowserContextAPIRequestContext(this); } isPersistentContext(): boolean { diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index 29d3ec403f473..c7913c8a0ebcb 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -44,9 +44,31 @@ type FetchRequestOptions = { baseURL?: string; }; +export type APIRequestEvent = { + url: URL, + method: string, + headers: { [name: string]: string }, + cookies: types.NameValueList, + postData?: Buffer +}; + +export type APIRequestFinishedEvent = { + requestEvent: APIRequestEvent, + httpVersion: string; + headers: http.IncomingHttpHeaders; + cookies: types.NetworkCookie[]; + rawHeaders: string[]; + statusCode: number; + statusMessage: string; + body?: Buffer; +}; + export abstract class APIRequestContext extends SdkObject { static Events = { Dispose: 'dispose', + + Request: 'request', + RequestFinished: 'requestfinished', }; readonly fetchResponses: Map = new Map(); @@ -166,7 +188,9 @@ export abstract class APIRequestContext extends SdkObject { return { ...fetchResponse, fetchUid }; } - private async _updateCookiesFromHeader(responseUrl: string, setCookie: string[]) { + private _parseSetCookieHeader(responseUrl: string, setCookie: string[] | undefined): types.NetworkCookie[] { + if (!setCookie) + return []; const url = new URL(responseUrl); // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4 const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/'); @@ -188,8 +212,7 @@ export abstract class APIRequestContext extends SdkObject { cookie.path = defaultPath; cookies.push(cookie); } - if (cookies.length) - await this._addCookies(cookies); + return cookies; } private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) { @@ -204,15 +227,43 @@ export abstract class APIRequestContext extends SdkObject { private async _sendRequest(progress: Progress, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise{ await this._updateRequestCookieHeader(url, options); + + const requestCookies = (options.headers!['cookie'] as (string | undefined))?.split(';').map(p => { + const [name, value] = p.split('=').map(v => v.trim()); + return { name, value }; + }) || []; + const requestEvent: APIRequestEvent = { + url, + method: options.method!, + headers: options.headers as { [name: string]: string }, + cookies: requestCookies, + postData + }; + this.emit(APIRequestContext.Events.Request, requestEvent); + return new Promise((fulfill, reject) => { const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest) = (url.protocol === 'https:' ? https : http).request; const request = requestConstructor(url, options, async response => { + const notifyRequestFinished = (body?: Buffer) => { + const requestFinishedEvent: APIRequestFinishedEvent = { + requestEvent, + statusCode: -1, + statusMessage: '', + ...response, + cookies, + body + }; + this.emit(APIRequestContext.Events.RequestFinished, requestFinishedEvent); + }; progress.log(`← ${response.statusCode} ${response.statusMessage}`); for (const [name, value] of Object.entries(response.headers)) progress.log(` ${name}: ${value}`); - if (response.headers['set-cookie']) - await this._updateCookiesFromHeader(response.url || url.toString(), response.headers['set-cookie']); + + const cookies = this._parseSetCookieHeader(response.url || url.toString(), response.headers['set-cookie']) ; + if (cookies.length) + await this._addCookies(cookies); + if (redirectStatus.includes(response.statusCode!)) { if (!options.maxRedirects) { reject(new Error('Max redirect count exceeded')); @@ -251,6 +302,7 @@ export abstract class APIRequestContext extends SdkObject { // HTTP-redirect fetch step 4: If locationURL is null, then return response. if (response.headers.location) { const locationURL = new URL(response.headers.location, url); + notifyRequestFinished(); fulfill(this._sendRequest(progress, locationURL, redirectOptions, postData)); request.destroy(); return; @@ -263,6 +315,7 @@ export abstract class APIRequestContext extends SdkObject { const { username, password } = credentials; const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64'); options.headers!['authorization'] = `Basic ${encoded}`; + notifyRequestFinished(); fulfill(this._sendRequest(progress, url, options, postData)); request.destroy(); return; @@ -294,6 +347,7 @@ export abstract class APIRequestContext extends SdkObject { body.on('data', chunk => chunks.push(chunk)); body.on('end', () => { const body = Buffer.concat(chunks); + notifyRequestFinished(body); fulfill({ url: response.url || url.toString(), status: response.statusCode || 0, diff --git a/packages/playwright-core/src/server/supplements/har/harTracer.ts b/packages/playwright-core/src/server/supplements/har/harTracer.ts index cad5fc75f0f91..07cfdf84ee447 100644 --- a/packages/playwright-core/src/server/supplements/har/harTracer.ts +++ b/packages/playwright-core/src/server/supplements/har/harTracer.ts @@ -15,6 +15,7 @@ */ import { BrowserContext } from '../../browserContext'; +import { APIRequestContext, APIRequestEvent, APIRequestFinishedEvent } from '../../fetch'; import { helper } from '../../helper'; import * as network from '../../network'; import { Page } from '../../page'; @@ -64,10 +65,12 @@ export class HarTracer { eventsHelper.addEventListener(this._context, BrowserContext.Events.Request, (request: network.Request) => this._onRequest(request)), eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFinished, ({ request, response }) => this._onRequestFinished(request, response).catch(() => {})), eventsHelper.addEventListener(this._context, BrowserContext.Events.Response, (response: network.Response) => this._onResponse(response)), + eventsHelper.addEventListener(this._context.fetchRequest, APIRequestContext.Events.Request, (event: APIRequestEvent) => this._onAPIRequest(event)), + eventsHelper.addEventListener(this._context.fetchRequest, APIRequestContext.Events.RequestFinished, (event: APIRequestFinishedEvent) => this._onAPIRequestFinished(event)), ]; } - private _entryForRequest(request: network.Request): har.Entry | undefined { + private _entryForRequest(request: network.Request | APIRequestEvent): har.Entry | undefined { return (request as any)[this._entrySymbol]; } @@ -132,6 +135,49 @@ export class HarTracer { this._barrierPromises.add(race); } + private _onAPIRequest(event: APIRequestEvent) { + const harEntry = createHarEntry(event.method, event.url, '', ''); + harEntry.request.cookies = event.cookies; + harEntry.request.headers = Object.entries(event.headers).map(([name, value]) => ({ name, value })); + harEntry.request.postData = postDataForBuffer(event.postData || null, event.headers['content-type'], this._options.content); + harEntry.request.bodySize = event.postData?.length || 0; + (event as any)[this._entrySymbol] = harEntry; + if (this._started) + this._delegate.onEntryStarted(harEntry); + } + + private _onAPIRequestFinished(event: APIRequestFinishedEvent): void { + const harEntry = this._entryForRequest(event.requestEvent); + if (!harEntry) + return; + + harEntry.response.status = event.statusCode; + harEntry.response.statusText = event.statusMessage; + harEntry.response.httpVersion = event.httpVersion; + harEntry.response.redirectURL = event.headers.location || ''; + for (let i = 0; i < event.rawHeaders.length; i += 2) { + harEntry.response.headers.push({ + name: event.rawHeaders[i], + value: event.rawHeaders[i + 1] + }); + } + harEntry.response.cookies = event.cookies.map(c => { + return { + ...c, + expires: c.expires === -1 ? undefined : new Date(c.expires) + }; + }); + + const content = harEntry.response.content; + const contentType = event.headers['content-type']; + if (contentType) + content.mimeType = contentType; + this._storeResponseContent(event.body, content); + + if (this._started) + this._delegate.onEntryFinished(harEntry); + } + private _onRequest(request: network.Request) { const page = request.frame()._page; const url = network.parsedURL(request.url()); @@ -139,49 +185,10 @@ export class HarTracer { return; const pageEntry = this._ensurePageEntry(page); - const harEntry: har.Entry = { - pageref: pageEntry.id, - _requestref: request.guid, - _frameref: request.frame().guid, - _monotonicTime: monotonicTime(), - startedDateTime: new Date(), - time: -1, - request: { - method: request.method(), - url: request.url(), - httpVersion: FALLBACK_HTTP_VERSION, - cookies: [], - headers: [], - queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })), - postData: postDataForHar(request, this._options.content), - headersSize: -1, - bodySize: request.bodySize(), - }, - response: { - status: -1, - statusText: '', - httpVersion: FALLBACK_HTTP_VERSION, - cookies: [], - headers: [], - content: { - size: -1, - mimeType: request.headerValue('content-type') || 'x-unknown', - }, - headersSize: -1, - bodySize: -1, - redirectURL: '', - _transferSize: -1 - }, - cache: { - beforeRequest: null, - afterRequest: null, - }, - timings: { - send: -1, - wait: -1, - receive: -1 - }, - }; + const harEntry = createHarEntry(request.method(), url, request.guid, request.frame().guid); + harEntry.pageref = pageEntry.id; + harEntry.request.postData = postDataForRequest(request, this._options.content); + harEntry.request.bodySize = request.bodySize(); if (request.redirectedFrom()) { const fromEntry = this._entryForRequest(request.redirectedFrom()!); if (fromEntry) @@ -232,18 +239,8 @@ export class HarTracer { } const content = harEntry.response.content; - content.size = buffer.length; compressionCalculationBarrier.setDecodedBodySize(buffer.length); - if (buffer && buffer.length > 0) { - if (this._options.content === 'embedded') { - content.text = buffer.toString('base64'); - content.encoding = 'base64'; - } else if (this._options.content === 'sha1') { - content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat'); - if (this._started) - this._delegate.onContentBlob(content._sha1, buffer); - } - } + this._storeResponseContent(buffer, content); }).catch(() => { compressionCalculationBarrier.setDecodedBodySize(0); }).then(() => { @@ -267,6 +264,22 @@ export class HarTracer { })); } + private _storeResponseContent(buffer: Buffer | undefined, content: har.Content) { + if (!buffer) { + content.size = 0; + return; + } + content.size = buffer.length; + if (this._options.content === 'embedded') { + content.text = buffer.toString('base64'); + content.encoding = 'base64'; + } else if (this._options.content === 'sha1') { + content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat'); + if (this._started) + this._delegate.onContentBlob(content._sha1, buffer); + } + } + private _onResponse(response: network.Response) { const page = response.frame()._page; const pageEntry = this._ensurePageEntry(page); @@ -275,7 +288,7 @@ export class HarTracer { return; const request = response.request(); - harEntry.request.postData = postDataForHar(request, this._options.content); + harEntry.request.postData = postDataForRequest(request, this._options.content); harEntry.response = { status: response.status(), @@ -372,12 +385,66 @@ export class HarTracer { } } -function postDataForHar(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined { +function createHarEntry(method: string, url: URL, requestref: string, frameref: string): har.Entry { + const harEntry: har.Entry = { + _requestref: requestref, + _frameref: frameref, + _monotonicTime: monotonicTime(), + startedDateTime: new Date(), + time: -1, + request: { + method: method, + url: url.toString(), + httpVersion: FALLBACK_HTTP_VERSION, + cookies: [], + headers: [], + queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })), + headersSize: -1, + bodySize: 0, + }, + response: { + status: -1, + statusText: '', + httpVersion: FALLBACK_HTTP_VERSION, + cookies: [], + headers: [], + content: { + size: -1, + mimeType: 'x-unknown', + }, + headersSize: -1, + bodySize: -1, + redirectURL: '', + _transferSize: -1 + }, + cache: { + beforeRequest: null, + afterRequest: null, + }, + timings: { + send: -1, + wait: -1, + receive: -1 + }, + }; + return harEntry; +} + +function postDataForRequest(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined { const postData = request.postDataBuffer(); if (!postData) return; - const contentType = request.headerValue('content-type') || 'application/octet-stream'; + const contentType = request.headerValue('content-type'); + return postDataForBuffer(postData, contentType, content); +} + +function postDataForBuffer(postData: Buffer | null, contentType: string | undefined, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined { + if (!postData) + return; + + contentType ??= 'application/octet-stream'; + const result: har.PostData = { mimeType: contentType, text: '', diff --git a/tests/har.spec.ts b/tests/har.spec.ts index ed79aaee4ed43..9e44b5e80ba3a 100644 --- a/tests/har.spec.ts +++ b/tests/har.spec.ts @@ -634,3 +634,40 @@ it('should include _requestref for redirects', async ({ contextFactory, server } expect(entryEmptyPage.request.url).toBe(server.EMPTY_PAGE); expect(entryEmptyPage._requestref).toBe(requests.get(entryEmptyPage.request.url)); }); + +it('should include API request', async ({ contextFactory, server }, testInfo) => { + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + const url = server.PREFIX + '/simple.json'; + const response = await page.request.post(url, { + headers: { cookie: 'a=b; c=d' }, + data: { foo: 'bar' } + }); + const responseBody = await response.body(); + const log = await getLog(); + expect(log.entries.length).toBe(1); + const entry = log.entries[0]; + expect(entry.request.url).toBe(url); + expect(entry.request.method).toBe('POST'); + expect(entry.request.httpVersion).toBe('HTTP/1.1'); + expect(entry.request.cookies).toEqual([ + { + 'name': 'a', + 'value': 'b' + }, + { + 'name': 'c', + 'value': 'd' + } + ]); + expect(entry.request.headers.length).toBeGreaterThan(1); + expect(entry.request.headers.find(h => h.name.toLowerCase() === 'user-agent')).toBeTruthy(); + expect(entry.request.headers.find(h => h.name.toLowerCase() === 'content-type')?.value).toBe('application/json'); + expect(entry.request.headers.find(h => h.name.toLowerCase() === 'content-length')?.value).toBe('13'); + expect(entry.request.bodySize).toBe(13); + + expect(entry.response.status).toBe(200); + expect(entry.response.headers.find(h => h.name.toLowerCase() === 'content-type')?.value).toContain('application/json'); + expect(entry.response.content.size).toBe(15); + expect(entry.response.content.text).toBe(responseBody.toString('base64')); +}); + diff --git a/tests/tracing.spec.ts b/tests/tracing.spec.ts index c7192ffcdb050..3c6fe0592043e 100644 --- a/tests/tracing.spec.ts +++ b/tests/tracing.spec.ts @@ -85,6 +85,19 @@ test('should exclude internal pages', async ({ browserName, context, page, serve expect(pageIds.size).toBe(1); }); +test('should include context API requests', async ({ browserName, context, page, server }, testInfo) => { + await context.tracing.start({ snapshots: true }); + await page.request.post(server.PREFIX + '/simple.json', { data: { foo: 'bar' } }); + await context.tracing.stop({ path: testInfo.outputPath('trace.zip') }); + const { events } = await parseTrace(testInfo.outputPath('trace.zip')); + const postEvent = events.find(e => e.metadata?.apiName === 'apiRequestContext.post'); + expect(postEvent).toBeTruthy(); + const harEntry = events.find(e => e.type === 'resource-snapshot'); + expect(harEntry).toBeTruthy(); + expect(harEntry.snapshot.request.url).toBe(server.PREFIX + '/simple.json'); + expect(harEntry.snapshot.response.status).toBe(200); +}); + test('should collect two traces', async ({ context, page, server }, testInfo) => { await context.tracing.start({ screenshots: true, snapshots: true }); await page.goto(server.EMPTY_PAGE);