Skip to content

Commit

Permalink
feat(tracing): trace context APIRequest calls (microsoft#10684)
Browse files Browse the repository at this point in the history
  • Loading branch information
@yury-s
yury-s committed Dec 2, 2021
1 parent 98e2f40 commit 8afd0b7
Show file tree
Hide file tree
Showing 5 changed files with 236 additions and 64 deletions.
3 changes: 2 additions & 1 deletion packages/playwright-core/src/server/browserContext.ts
View file
Expand Up @@ -78,11 +78,12 @@ export abstract class BrowserContext extends SdkObject {
// Create instrumentation per context.
this.instrumentation = createInstrumentation();

this.fetchRequest = new BrowserContextAPIRequestContext(this);

if (this._options.recordHar)
this._harRecorder = new HarRecorder(this, { ...this._options.recordHar, path: path.join(this._browser.options.artifactsDir, `${createGuid()}.har`) });

this.tracing = new Tracing(this);
this.fetchRequest = new BrowserContextAPIRequestContext(this);
}

isPersistentContext(): boolean {
Expand Down
64 changes: 59 additions & 5 deletions packages/playwright-core/src/server/fetch.ts
View file
Expand Up @@ -44,9 +44,31 @@ type FetchRequestOptions = {
baseURL?: string;
};

export type APIRequestEvent = {
url: URL,
method: string,
headers: { [name: string]: string },
cookies: types.NameValueList,
postData?: Buffer
};

export type APIRequestFinishedEvent = {
requestEvent: APIRequestEvent,
httpVersion: string;
headers: http.IncomingHttpHeaders;
cookies: types.NetworkCookie[];
rawHeaders: string[];
statusCode: number;
statusMessage: string;
body?: Buffer;
};

export abstract class APIRequestContext extends SdkObject {
static Events = {
Dispose: 'dispose',

Request: 'request',
RequestFinished: 'requestfinished',
};

readonly fetchResponses: Map<string, Buffer> = new Map();
Expand Down Expand Up @@ -166,7 +188,9 @@ export abstract class APIRequestContext extends SdkObject {
return { ...fetchResponse, fetchUid };
}

private async _updateCookiesFromHeader(responseUrl: string, setCookie: string[]) {
private _parseSetCookieHeader(responseUrl: string, setCookie: string[] | undefined): types.NetworkCookie[] {
if (!setCookie)
return [];
const url = new URL(responseUrl);
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
Expand All @@ -188,8 +212,7 @@ export abstract class APIRequestContext extends SdkObject {
cookie.path = defaultPath;
cookies.push(cookie);
}
if (cookies.length)
await this._addCookies(cookies);
return cookies;
}

private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) {
Expand All @@ -204,15 +227,43 @@ export abstract class APIRequestContext extends SdkObject {

private async _sendRequest(progress: Progress, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.APIResponse>{
await this._updateRequestCookieHeader(url, options);

const requestCookies = (options.headers!['cookie'] as (string | undefined))?.split(';').map(p => {
const [name, value] = p.split('=').map(v => v.trim());
return { name, value };
}) || [];
const requestEvent: APIRequestEvent = {
url,
method: options.method!,
headers: options.headers as { [name: string]: string },
cookies: requestCookies,
postData
};
this.emit(APIRequestContext.Events.Request, requestEvent);

return new Promise<types.APIResponse>((fulfill, reject) => {
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
= (url.protocol === 'https:' ? https : http).request;
const request = requestConstructor(url, options, async response => {
const notifyRequestFinished = (body?: Buffer) => {
const requestFinishedEvent: APIRequestFinishedEvent = {
requestEvent,
statusCode: -1,
statusMessage: '',
...response,
cookies,
body
};
this.emit(APIRequestContext.Events.RequestFinished, requestFinishedEvent);
};
progress.log(`← ${response.statusCode} ${response.statusMessage}`);
for (const [name, value] of Object.entries(response.headers))
progress.log(` ${name}: ${value}`);
if (response.headers['set-cookie'])
await this._updateCookiesFromHeader(response.url || url.toString(), response.headers['set-cookie']);

const cookies = this._parseSetCookieHeader(response.url || url.toString(), response.headers['set-cookie']) ;
if (cookies.length)
await this._addCookies(cookies);

if (redirectStatus.includes(response.statusCode!)) {
if (!options.maxRedirects) {
reject(new Error('Max redirect count exceeded'));
Expand Down Expand Up @@ -251,6 +302,7 @@ export abstract class APIRequestContext extends SdkObject {
// HTTP-redirect fetch step 4: If locationURL is null, then return response.
if (response.headers.location) {
const locationURL = new URL(response.headers.location, url);
notifyRequestFinished();
fulfill(this._sendRequest(progress, locationURL, redirectOptions, postData));
request.destroy();
return;
Expand All @@ -263,6 +315,7 @@ export abstract class APIRequestContext extends SdkObject {
const { username, password } = credentials;
const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64');
options.headers!['authorization'] = `Basic ${encoded}`;
notifyRequestFinished();
fulfill(this._sendRequest(progress, url, options, postData));
request.destroy();
return;
Expand Down Expand Up @@ -294,6 +347,7 @@ export abstract class APIRequestContext extends SdkObject {
body.on('data', chunk => chunks.push(chunk));
body.on('end', () => {
const body = Buffer.concat(chunks);
notifyRequestFinished(body);
fulfill({
url: response.url || url.toString(),
status: response.statusCode || 0,
Expand Down
183 changes: 125 additions & 58 deletions packages/playwright-core/src/server/supplements/har/harTracer.ts
View file
Expand Up @@ -15,6 +15,7 @@
*/

import { BrowserContext } from '../../browserContext';
import { APIRequestContext, APIRequestEvent, APIRequestFinishedEvent } from '../../fetch';
import { helper } from '../../helper';
import * as network from '../../network';
import { Page } from '../../page';
Expand Down Expand Up @@ -64,10 +65,12 @@ export class HarTracer {
eventsHelper.addEventListener(this._context, BrowserContext.Events.Request, (request: network.Request) => this._onRequest(request)),
eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFinished, ({ request, response }) => this._onRequestFinished(request, response).catch(() => {})),
eventsHelper.addEventListener(this._context, BrowserContext.Events.Response, (response: network.Response) => this._onResponse(response)),
eventsHelper.addEventListener(this._context.fetchRequest, APIRequestContext.Events.Request, (event: APIRequestEvent) => this._onAPIRequest(event)),
eventsHelper.addEventListener(this._context.fetchRequest, APIRequestContext.Events.RequestFinished, (event: APIRequestFinishedEvent) => this._onAPIRequestFinished(event)),
];
}

private _entryForRequest(request: network.Request): har.Entry | undefined {
private _entryForRequest(request: network.Request | APIRequestEvent): har.Entry | undefined {
return (request as any)[this._entrySymbol];
}

Expand Down Expand Up @@ -132,56 +135,60 @@ export class HarTracer {
this._barrierPromises.add(race);
}

private _onAPIRequest(event: APIRequestEvent) {
const harEntry = createHarEntry(event.method, event.url, '', '');
harEntry.request.cookies = event.cookies;
harEntry.request.headers = Object.entries(event.headers).map(([name, value]) => ({ name, value }));
harEntry.request.postData = postDataForBuffer(event.postData || null, event.headers['content-type'], this._options.content);
harEntry.request.bodySize = event.postData?.length || 0;
(event as any)[this._entrySymbol] = harEntry;
if (this._started)
this._delegate.onEntryStarted(harEntry);
}

private _onAPIRequestFinished(event: APIRequestFinishedEvent): void {
const harEntry = this._entryForRequest(event.requestEvent);
if (!harEntry)
return;

harEntry.response.status = event.statusCode;
harEntry.response.statusText = event.statusMessage;
harEntry.response.httpVersion = event.httpVersion;
harEntry.response.redirectURL = event.headers.location || '';
for (let i = 0; i < event.rawHeaders.length; i += 2) {
harEntry.response.headers.push({
name: event.rawHeaders[i],
value: event.rawHeaders[i + 1]
});
}
harEntry.response.cookies = event.cookies.map(c => {
return {
...c,
expires: c.expires === -1 ? undefined : new Date(c.expires)
};
});

const content = harEntry.response.content;
const contentType = event.headers['content-type'];
if (contentType)
content.mimeType = contentType;
this._storeResponseContent(event.body, content);

if (this._started)
this._delegate.onEntryFinished(harEntry);
}

private _onRequest(request: network.Request) {
const page = request.frame()._page;
const url = network.parsedURL(request.url());
if (!url)
return;

const pageEntry = this._ensurePageEntry(page);
const harEntry: har.Entry = {
pageref: pageEntry.id,
_requestref: request.guid,
_frameref: request.frame().guid,
_monotonicTime: monotonicTime(),
startedDateTime: new Date(),
time: -1,
request: {
method: request.method(),
url: request.url(),
httpVersion: FALLBACK_HTTP_VERSION,
cookies: [],
headers: [],
queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
postData: postDataForHar(request, this._options.content),
headersSize: -1,
bodySize: request.bodySize(),
},
response: {
status: -1,
statusText: '',
httpVersion: FALLBACK_HTTP_VERSION,
cookies: [],
headers: [],
content: {
size: -1,
mimeType: request.headerValue('content-type') || 'x-unknown',
},
headersSize: -1,
bodySize: -1,
redirectURL: '',
_transferSize: -1
},
cache: {
beforeRequest: null,
afterRequest: null,
},
timings: {
send: -1,
wait: -1,
receive: -1
},
};
const harEntry = createHarEntry(request.method(), url, request.guid, request.frame().guid);
harEntry.pageref = pageEntry.id;
harEntry.request.postData = postDataForRequest(request, this._options.content);
harEntry.request.bodySize = request.bodySize();
if (request.redirectedFrom()) {
const fromEntry = this._entryForRequest(request.redirectedFrom()!);
if (fromEntry)
Expand Down Expand Up @@ -232,18 +239,8 @@ export class HarTracer {
}

const content = harEntry.response.content;
content.size = buffer.length;
compressionCalculationBarrier.setDecodedBodySize(buffer.length);
if (buffer && buffer.length > 0) {
if (this._options.content === 'embedded') {
content.text = buffer.toString('base64');
content.encoding = 'base64';
} else if (this._options.content === 'sha1') {
content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
if (this._started)
this._delegate.onContentBlob(content._sha1, buffer);
}
}
this._storeResponseContent(buffer, content);
}).catch(() => {
compressionCalculationBarrier.setDecodedBodySize(0);
}).then(() => {
Expand All @@ -267,6 +264,22 @@ export class HarTracer {
}));
}

private _storeResponseContent(buffer: Buffer | undefined, content: har.Content) {
if (!buffer) {
content.size = 0;
return;
}
content.size = buffer.length;
if (this._options.content === 'embedded') {
content.text = buffer.toString('base64');
content.encoding = 'base64';
} else if (this._options.content === 'sha1') {
content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
if (this._started)
this._delegate.onContentBlob(content._sha1, buffer);
}
}

private _onResponse(response: network.Response) {
const page = response.frame()._page;
const pageEntry = this._ensurePageEntry(page);
Expand All @@ -275,7 +288,7 @@ export class HarTracer {
return;
const request = response.request();

harEntry.request.postData = postDataForHar(request, this._options.content);
harEntry.request.postData = postDataForRequest(request, this._options.content);

harEntry.response = {
status: response.status(),
Expand Down Expand Up @@ -372,12 +385,66 @@ export class HarTracer {
}
}

function postDataForHar(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
function createHarEntry(method: string, url: URL, requestref: string, frameref: string): har.Entry {
const harEntry: har.Entry = {
_requestref: requestref,
_frameref: frameref,
_monotonicTime: monotonicTime(),
startedDateTime: new Date(),
time: -1,
request: {
method: method,
url: url.toString(),
httpVersion: FALLBACK_HTTP_VERSION,
cookies: [],
headers: [],
queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
headersSize: -1,
bodySize: 0,
},
response: {
status: -1,
statusText: '',
httpVersion: FALLBACK_HTTP_VERSION,
cookies: [],
headers: [],
content: {
size: -1,
mimeType: 'x-unknown',
},
headersSize: -1,
bodySize: -1,
redirectURL: '',
_transferSize: -1
},
cache: {
beforeRequest: null,
afterRequest: null,
},
timings: {
send: -1,
wait: -1,
receive: -1
},
};
return harEntry;
}

function postDataForRequest(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
const postData = request.postDataBuffer();
if (!postData)
return;

const contentType = request.headerValue('content-type') || 'application/octet-stream';
const contentType = request.headerValue('content-type');
return postDataForBuffer(postData, contentType, content);
}

function postDataForBuffer(postData: Buffer | null, contentType: string | undefined, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
if (!postData)
return;

contentType ??= 'application/octet-stream';

const result: har.PostData = {
mimeType: contentType,
text: '',
Expand Down

0 comments on commit 8afd0b7

Please sign in to comment.