From 10dd270c869ff7fa54c7a0cf89adde09ceb1d7d9 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 21 Aug 2026 16:09:36 -0700 Subject: [PATCH] feat(reporter): report step params in the chrome://tracing report Step params reported to the reporters land in the `params` trace event argument of the corresponding slice. Render the API call params through a class/method-aware switch keyed the same way as the protocol metainfo, so that each call reports the arguments that say what it did. Unbounded arguments are never reported and long strings are truncated to keep the reports small. Write the report event by event instead of serializing it into a single string, and stream it through gzip when the output file ends with ".gz". --- docs/src/test-reporter-api/class-teststep.md | 7 ++ docs/src/test-reporters-js.md | 7 +- packages/playwright/src/index.ts | 117 ++++++++++++++++-- .../playwright/src/reporters/chromeTrace.ts | 71 ++++++++--- packages/playwright/types/testReporter.d.ts | 6 + .../reporter-chrome-trace.spec.ts | 45 ++++++- tests/playwright-test/test-step.spec.ts | 79 ++++++++++++ 7 files changed, 304 insertions(+), 28 deletions(-) diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index f437bab4d7f23..8ff66f4be55d1 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -56,12 +56,19 @@ await page.getByRole('button').click(); // { url: 'https://example.com' } await page.goto('https://example.com'); +// { locator: 'getByLabel(\'Password\')', value: 'secret' } +await page.getByLabel('Password').fill('secret'); + // { orderId: 42 } await test.step('checkout', async () => { // ... }, { params: { orderId: 42 } }); ``` +To keep the reports small, Playwright API calls only report a curated set of arguments per call, and long string values +are truncated. Unbounded arguments such as the page content, evaluated expressions or request bodies are never +reported. + ## property: TestStep.startTime * since: v1.10 - type: <[Date]> diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 0e049b8c6f84d..c80500f3b0c4f 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -433,19 +433,20 @@ JUnit report supports following configuration options and environment variables: ### Chrome tracing reporter -Chrome tracing reporter produces a [Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview) json file that can be opened in [`chrome://tracing`](chrome://tracing) or in the [Perfetto UI](https://ui.perfetto.dev). It renders the test run as a timeline with a lane per worker, where every test is a slice containing its before/after hooks, fixtures and steps as nested slices. Every slice carries the details of the test or step in its trace event arguments, including source locations, tags, annotations, errors, stdio and paths to the attachment files. +Chrome tracing reporter produces a [Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview) json file that can be opened in [`chrome://tracing`](chrome://tracing) or in the [Perfetto UI](https://ui.perfetto.dev). It renders the test run as a timeline with a lane per worker, where every test is a slice containing its before/after hooks, fixtures and steps as nested slices. Every slice carries the details of the test or step in its trace event arguments, including source locations, [`property: TestStep.params`], tags, annotations, errors, stdio and paths to the attachment files. ```bash npx playwright test --reporter=chrome-trace ``` -By default the report is written to `test-results/chrome-trace.json`. +By default the report is written to `test-results/chrome-trace.json`. When the output file name ends with `.gz`, the +report is gzipped on the fly, which both viewers accept and which is worth it for large test runs. ```js title="playwright.config.ts" import { defineConfig } from '@playwright/test'; export default defineConfig({ - reporter: [['chrome-trace', { outputFile: 'chrome-trace.json' }]], + reporter: [['chrome-trace', { outputFile: 'chrome-trace.json.gz' }]], }); ``` diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 6e61a76a85b10..eda97b3fa4d40 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -112,7 +112,7 @@ const utilityFixtures: Fixtures = { } // In the general case, create a step for each api call and connect them through the stepId. - const params = renderParams(channel.params); + const params = renderParams(channel.type, channel.method, channel.params); const step = testInfo._addStep({ location: data.frames[0], category: 'pw:api', @@ -925,23 +925,120 @@ function renderTitle(type: string, method: string, params: Record): Record | undefined { + switch (`${type}.${method}`) { + case 'APIRequestContext.fetch': + return { url: params.url, method: params.method }; + case 'Frame.goto': + case 'Frame.addScriptTag': + case 'Frame.addStyleTag': + return { url: params.url }; + + case 'Frame.click': + case 'Frame.dblclick': + case 'ElementHandle.click': + case 'ElementHandle.dblclick': + return { button: params.button, clickCount: params.clickCount, modifiers: params.modifiers, position: params.position }; + case 'Frame.hover': + case 'Frame.tap': + case 'ElementHandle.hover': + case 'ElementHandle.tap': + return { modifiers: params.modifiers, position: params.position }; + case 'Frame.check': + case 'Frame.uncheck': + case 'ElementHandle.check': + case 'ElementHandle.uncheck': + return { position: params.position }; + case 'Frame.dragAndDrop': + return { source: renderLocator(params.source), target: renderLocator(params.target) }; + case 'Frame.fill': + case 'ElementHandle.fill': + return { value: params.value }; + case 'Frame.press': + case 'ElementHandle.press': + case 'Page.keyboardDown': + case 'Page.keyboardUp': + case 'Page.keyboardPress': + return { key: params.key }; + case 'Frame.type': + case 'ElementHandle.type': + case 'Page.keyboardType': + case 'Page.keyboardInsertText': + return { text: params.text }; + case 'Frame.dispatchEvent': + case 'ElementHandle.dispatchEvent': + return { type: params.type }; + case 'Frame.selectOption': + case 'ElementHandle.selectOption': + return { options: params.options }; + case 'Frame.setInputFiles': + case 'ElementHandle.setInputFiles': + return { files: params.localPaths }; + + case 'Page.mouseClick': + return { x: params.x, y: params.y, button: params.button, clickCount: params.clickCount }; + case 'Page.mouseMove': + case 'Page.touchscreenTap': + return { x: params.x, y: params.y }; + case 'Page.mouseDown': + case 'Page.mouseUp': + return { button: params.button, clickCount: params.clickCount }; + case 'Page.mouseWheel': + return { deltaX: params.deltaX, deltaY: params.deltaY }; + + case 'Frame.waitForSelector': + case 'ElementHandle.waitForSelector': + case 'ElementHandle.waitForElementState': + return { state: params.state }; + case 'Frame.waitForTimeout': + return { timeout: params.waitTimeout }; + + case 'Page.emulateMedia': + return { media: params.media, colorScheme: params.colorScheme, reducedMotion: params.reducedMotion, forcedColors: params.forcedColors, contrast: params.contrast }; + case 'Page.setViewportSize': + return params.viewportSize; + case 'Page.screenshot': + case 'ElementHandle.screenshot': + return { type: params.type, fullPage: params.fullPage }; + case 'BrowserContext.setOffline': + return { offline: params.offline }; + case 'Dialog.accept': + return { promptText: params.promptText }; + case 'Tracing.tracingGroup': + return { name: params.name }; + } +} -function renderParams(params: Record | undefined): Record | undefined { +function renderParams(type: string, method: string, params: Record | undefined): Record | undefined { if (!params) return undefined; const result: Record = {}; - for (const [name, value] of Object.entries(params)) { - if (!kIncludedParams.has(name)) - continue; - if (name === 'selector' && typeof value === 'string') - result.locator = asLocatorDescription('javascript', value); - else if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') - result[name] = value; + const locator = renderLocator(params.selector); + if (locator !== undefined) + result.locator = locator; + for (const [name, value] of Object.entries(renderCallParams(type, method, params) ?? {})) { + if (value !== undefined) + result[name] = typeof value === 'string' ? truncateParam(value) : value; } return Object.keys(result).length ? result : undefined; } +function renderLocator(selector: any): string | undefined { + if (typeof selector !== 'string') + return undefined; + return truncateParam(asLocatorDescription('javascript', selector)); +} + +function truncateParam(value: string): string { + return value.length > kMaxParamLength ? value.substring(0, kMaxParamLength) + '\u2026' : value; +} + function tracing() { return (test.info() as TestInfoImpl)._tracing; } diff --git a/packages/playwright/src/reporters/chromeTrace.ts b/packages/playwright/src/reporters/chromeTrace.ts index de268ea5aac10..b3def8c83f882 100644 --- a/packages/playwright/src/reporters/chromeTrace.ts +++ b/packages/playwright/src/reporters/chromeTrace.ts @@ -16,6 +16,7 @@ import fs from 'fs'; import path from 'path'; +import zlib from 'zlib'; import { toPosixPath } from '@utils/fileUtils'; import { getPlaywrightVersion } from 'playwright-core/lib/coreBundle'; @@ -24,6 +25,7 @@ import { formatError, nonTerminalScreen, resolveOutputFile, CommonReporterOption import { stripAnsiEscapes } from '../util'; import type { ReporterV2 } from './reporterV2'; +import type { Writable } from 'stream'; import type { ChromeTraceReporterOptions } from '../../types/test'; import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; @@ -209,6 +211,8 @@ class ChromeTraceReporter implements ReporterV2 { private _stepArgs(step: TestStep): Record | undefined { const args: Record = {}; + if (step.params) + args.params = step.params; if (step.location) args.location = this._formatLocation(step.location); if (step.annotations.length) @@ -243,13 +247,13 @@ class ChromeTraceReporter implements ReporterV2 { for (const event of this._events) timeOrigin = Math.min(timeOrigin, event.ts); - const traceEvents: TraceEvent[] = [ + const metadataEvents: TraceEvent[] = [ { name: 'process_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { name: 'Playwright Test' } }, { name: 'process_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { sort_index: 0 } }, ]; for (const lane of [...this._laneEndTime.keys()].sort((a, b) => a - b)) { - traceEvents.push({ name: 'thread_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { name: `Worker ${lane}` } }); - traceEvents.push({ name: 'thread_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { sort_index: lane + 1 } }); + metadataEvents.push({ name: 'thread_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { name: `Worker ${lane}` } }); + metadataEvents.push({ name: 'thread_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { sort_index: lane + 1 } }); } // Sort is stable, so parent slices stay ahead of their children on ties. @@ -258,22 +262,61 @@ class ChromeTraceReporter implements ReporterV2 { event.ts = Math.round((event.ts - timeOrigin) * 1000); if (event.dur !== undefined) event.dur = Math.round(event.dur * 1000); - traceEvents.push(event); } - const report = { - traceEvents, - displayTimeUnit: 'ms', - metadata: { - 'playwright-version': getPlaywrightVersion(), - 'start-time': result.startTime.toISOString(), - 'duration': result.duration, - 'status': result.status, - }, + const metadata = { + 'playwright-version': getPlaywrightVersion(), + 'start-time': result.startTime.toISOString(), + 'duration': result.duration, + 'status': result.status, }; + // Serialize event by event, the whole report does not have to fit into memory twice. await fs.promises.mkdir(path.dirname(this._resolvedOutputFile), { recursive: true }); - await fs.promises.writeFile(this._resolvedOutputFile, JSON.stringify(report)); + const writer = new ChunkWriter(this._resolvedOutputFile); + await writer.write('{"traceEvents":['); + let separator = ''; + for (const events of [metadataEvents, this._events]) { + for (const event of events) { + await writer.write(separator + JSON.stringify(event)); + separator = ','; + } + } + await writer.write(`],"displayTimeUnit":"ms","metadata":${JSON.stringify(metadata)}}`); + await writer.close(); + } +} + +// Writes into a ".gz" file through a gzip stream, into a plain file otherwise. +class ChunkWriter { + private _stream: Writable; + private _closed: Promise; + private _error: Error | undefined; + + constructor(file: string) { + const fileStream = fs.createWriteStream(file); + const gzip = file.endsWith('.gz') ? zlib.createGzip() : undefined; + gzip?.pipe(fileStream); + this._stream = gzip ?? fileStream; + // The file is only complete once the destination closes, which is later than + // the gzip stream ending. + this._closed = new Promise(resolve => fileStream.on('close', resolve)); + for (const stream of new Set([this._stream, fileStream])) + stream.on('error', error => this._error ??= error); + } + + async write(chunk: string) { + if (this._error) + throw this._error; + if (!this._stream.write(chunk)) + await new Promise(resolve => this._stream.once('drain', () => resolve())); + } + + async close() { + this._stream.end(); + await this._closed; + if (this._error) + throw this._error; } } diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index a9634b136a82f..0b84a322939e1 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -888,12 +888,18 @@ export interface TestStep { * // { url: 'https://example.com' } * await page.goto('https://example.com'); * + * // { locator: 'getByLabel(\'Password\')', value: 'secret' } + * await page.getByLabel('Password').fill('secret'); + * * // { orderId: 42 } * await test.step('checkout', async () => { * // ... * }, { params: { orderId: 42 } }); * ``` * + * To keep the reports small, Playwright API calls only report a curated set of arguments per call, and long string + * values are truncated. Unbounded arguments such as the page content, evaluated expressions or request bodies are + * never reported. */ params?: { [key: string]: any; }; diff --git a/tests/playwright-test/reporter-chrome-trace.spec.ts b/tests/playwright-test/reporter-chrome-trace.spec.ts index 77f3ca83f8520..6a9193b6364f4 100644 --- a/tests/playwright-test/reporter-chrome-trace.spec.ts +++ b/tests/playwright-test/reporter-chrome-trace.spec.ts @@ -16,6 +16,7 @@ import * as fs from 'fs'; import * as path from 'path'; +import * as zlib from 'zlib'; import { test, expect } from './playwright-test-fixtures'; type TraceEvent = { @@ -31,7 +32,9 @@ type TraceEvent = { }; function readTrace(baseDir: string, fileName: string = 'test-results/chrome-trace.json') { - return JSON.parse(fs.readFileSync(path.join(baseDir, fileName), 'utf8')) as { + const file = path.join(baseDir, fileName); + const content = fileName.endsWith('.gz') ? zlib.gunzipSync(fs.readFileSync(file)).toString('utf8') : fs.readFileSync(file, 'utf8'); + return JSON.parse(content) as { traceEvents: TraceEvent[], displayTimeUnit: string, metadata: any, @@ -189,6 +192,29 @@ test('should report attachment files', async ({ runInlineTest }, testInfo) => { expect(one.args.attachments).toEqual(['inline', expect.stringMatching(/^test-results\/a-one\/attachments\/file-.*\.txt$/)]); }); +test('should report step params', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({ page }) => { + await page.goto('about:blank'); + await page.setContent(''); + await page.getByRole('button').click(); + await expect(page.getByRole('button')).toBeVisible(); + await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } }); + }); + `, + }, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(0); + + const events = slices(readTrace(testInfo.outputPath()).traceEvents); + expect(findSlice(events, 'Navigate to "about:blank"')!.args.params).toEqual({ url: 'about:blank' }); + expect(findSlice(events, 'Set content')!.args.params).toBe(undefined); + expect(findSlice(events, `Click getByRole('button')`)!.args.params).toEqual({ locator: `getByRole('button')` }); + expect(findSlice(events, `Expect "toBeVisible" getByRole('button')`)!.args.params).toEqual({ locator: `getByRole('button')` }); + expect(findSlice(events, 'my step')!.args.params).toEqual({ foo: 'bar', count: 7 }); +}); + test('should respect outputFile option', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'playwright.config.ts': ` @@ -203,6 +229,23 @@ test('should respect outputFile option', async ({ runInlineTest }, testInfo) => expect(findSlice(readTrace(testInfo.outputPath(), 'reports/my-trace.json').traceEvents, 'one')).toBeTruthy(); }); +test('should gzip the report when output file ends with .gz', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { reporter: [['chrome-trace', { outputFile: 'chrome-trace.json.gz' }]] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + }); + expect(result.exitCode).toBe(0); + + const gzipped = fs.readFileSync(testInfo.outputPath('chrome-trace.json.gz')); + expect(gzipped.subarray(0, 2)).toEqual(Buffer.from([0x1f, 0x8b])); + expect(findSlice(readTrace(testInfo.outputPath(), 'chrome-trace.json.gz').traceEvents, 'one')).toBeTruthy(); +}); + test('should respect PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'a.test.ts': ` diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts index 15a8b68116836..da821b6f25109 100644 --- a/tests/playwright-test/test-step.spec.ts +++ b/tests/playwright-test/test-step.spec.ts @@ -1897,3 +1897,82 @@ test('should report step params', async ({ runInlineTest }) => { `test.step | my step | {"foo":"bar","count":7}`, ]); }); + +test('should report input step params', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter'; + export default class MyReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.location?.file.endsWith('a.test.ts')) + console.log('%%' + step.title + ' | ' + JSON.stringify(step.params)); + } + } + `, + 'playwright.config.ts': ` + module.exports = { reporter: './reporter' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent(''); + await page.locator('#i').fill('value'); + await page.locator('#i').press('Enter'); + await page.keyboard.type('typed'); + await page.getByRole('button').click({ button: 'right', clickCount: 2, modifiers: ['Shift'], position: { x: 3, y: 4 } }); + await page.mouse.move(10, 20); + await page.mouse.wheel(0, 100); + await page.locator('select').selectOption('b'); + await page.dispatchEvent('#i', 'focus'); + await page.locator('#i').waitFor({ state: 'visible' }); + await page.setViewportSize({ width: 800, height: 600 }); + await page.dragAndDrop('#i', 'select'); + await page.evaluate(() => 1); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `Set content | undefined`, + `Fill "value" locator('#i') | {"locator":"locator('#i')","value":"value"}`, + `Press "Enter" locator('#i') | {"locator":"locator('#i')","key":"Enter"}`, + `Type "typed" | {"text":"typed"}`, + `Click getByRole('button') | {"locator":"getByRole('button')","button":"right","clickCount":2,"modifiers":["Shift"],"position":{"x":3,"y":4}}`, + `Mouse move | {"x":10,"y":20}`, + `Mouse wheel | {"deltaX":0,"deltaY":100}`, + `Select option locator('select') | {"locator":"locator('select')","options":[{"valueOrLabel":"b"}]}`, + `Dispatch "focus" locator('#i') | {"locator":"locator('#i')","type":"focus"}`, + `Wait for selector locator('#i') | {"locator":"locator('#i')","state":"visible"}`, + `Set viewport size | {"width":800,"height":600}`, + `Drag and drop | {"source":"locator('#i')","target":"locator('select')"}`, + `Evaluate | undefined`, + ]); +}); + +test('should truncate long step params', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter'; + export default class MyReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.params?.value) + console.log('%%' + step.params.value); + } + } + `, + 'playwright.config.ts': ` + module.exports = { reporter: './reporter' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent(''); + await page.locator('#i').fill('x'.repeat(1000)); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual(['x'.repeat(200) + '\u2026']); +});