diff --git a/README.md b/README.md index 26fb799..23309c0 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ reporter: [ outputFile: 'custom-name.json', // Optional: Output file name. Defaults to 'ctrf-report.json'. outputDir: 'custom-directory', // Optional: Output directory path. Defaults to '.' (project root). minimal: true, // Optional: Generate a minimal report. Defaults to 'false'. Overrides screenshot and testType when set to true + testStepsOnly: false, // Optional: Include only 'test.steps' category steps in the report. Read 'Advanced Usage' section below for details. screenshot: false, // Optional: Include screenshots in the report. Defaults to 'false'. annotations: false, // Optional: Include annotations in the report. Defaults to 'false'. testType: 'e2e', // Optional: Specify the test type (e.g., 'api', 'e2e'). Defaults to 'e2e'. @@ -136,30 +137,68 @@ Replace directory with the path to the directory containing the CTRF reports you The test object in the report includes the following [CTRF properties](https://ctrf.io/docs/schema/test): -| Name | Type | Required | Details | -| ------------ | ---------------- | -------- | ----------------------------------------------------------------------------------- | -| `name` | String | Required | The name of the test. | -| `status` | String | Required | The outcome of the test. One of: `passed`, `failed`, `skipped`, `pending`, `other`. | -| `duration` | Number | Required | The time taken for the test execution, in milliseconds. | -| `start` | Number | Optional | The start time of the test as a Unix epoch timestamp. | -| `stop` | Number | Optional | The end time of the test as a Unix epoch timestamp. | -| `suite` | String | Optional | The suite or group to which the test belongs. | -| `message` | String | Optional | The failure message if the test failed. | -| `trace` | String | Optional | The stack trace captured if the test failed. | -| `rawStatus` | String | Optional | The original playwright status of the test before mapping to CTRF status. | -| `tags` | Array of Strings | Optional | The tags retrieved from the test name | -| `type` | String | Optional | The type of test (e.g., `api`, `e2e`). | -| `filepath` | String | Optional | The file path where the test is located in the project. | -| `retries` | Number | Optional | The number of retries attempted for the test. | -| `flaky` | Boolean | Optional | Indicates whether the test result is flaky. | -| `browser` | String | Optional | The browser used for the test. | -| `screenshot` | String | Optional | A base64 encoded screenshot taken during the test. | -| `steps` | Array of Objects | Optional | Individual steps in the test, especially for BDD-style testing. | +| Name | Type | Required | Details | +| ------------ | ---------------- | -------- | ---------------------------------------------------------------------------------- | +| `name` | String | Required | The name of the test. | +| `status` | String | Required | The outcome of the test. One of:`passed`, `failed`, `skipped`, `pending`, `other`. | +| `duration` | Number | Required | The time taken for the test execution, in milliseconds. | +| `start` | Number | Optional | The start time of the test as a Unix epoch timestamp. | +| `stop` | Number | Optional | The end time of the test as a Unix epoch timestamp. | +| `suite` | String | Optional | The suite or group to which the test belongs. | +| `message` | String | Optional | The failure message if the test failed. | +| `trace` | String | Optional | The stack trace captured if the test failed. | +| `rawStatus` | String | Optional | The original playwright status of the test before mapping to CTRF status. | +| `tags` | Array of Strings | Optional | The tags retrieved from the test name | +| `type` | String | Optional | The type of test (e.g.,`api`, `e2e`). | +| `filepath` | String | Optional | The file path where the test is located in the project. | +| `retries` | Number | Optional | The number of retries attempted for the test. | +| `flaky` | Boolean | Optional | Indicates whether the test result is flaky. | +| `browser` | String | Optional | The browser used for the test. | +| `screenshot` | String | Optional | A base64 encoded screenshot taken during the test. | +| `steps` | Array of Objects | Optional | Individual steps in the test, especially for BDD-style testing. | +| `extra` | Object | Optional | Custom data relevant to the test. | + +## BDD styled tests + +⚠️ BDD styled tests that use [Cucumber](https://cucumber.io/docs/guides/overview/) as test runner are not supported. Only [playwright-bdd](https://vitalets.github.io/playwright-bdd/#/) is supported because it uses [Playwright Test](https://playwright.dev/docs/test-configuration) as a test runner to execute the tests. ## Advanced usage Some features require additional setup or usage considerations. +### Test steps + +Test steps are included in the `steps` property of a `Test` object. However, this `steps` property includes only top level steps. This means any children steps i.e. implementation details, are not included in it. + +Children steps are included in the `extra` property as `childSteps`. + +Every step has following properties of its own. + +| Name | Type | Required | Details | +| -------- | ------ | -------- | ---------------------------------------------------------------------------------- | +| `name` | String | Required | The name/title of the step. | +| `status` | String | Required | The outcome of the step. One of:`passed`, `failed`, `skipped`, `pending`, `other`. | +| `extra` | Object | Optional | Custom data relevant to the step. | + +Apart from including child steps, `extra` property is also used to report a steps' category, execution duration and the location in a source file. `extra` object contains following additional properties generated specifically for this JSON report. + +| Name | Type | Details | +| ------------ | --------------- | --------------------------------------------------------------------------------------------- | +| `category` | String | The category of the step. One of:`hook`, `expect`, `pw:api`, `test.step` | +| `duration` | String | The execution duration of the step, in milliseconds. | +| `location` | Object | Location the step in test suite, including file name, line number and column number. | +| `childSteps` | Array of object | Child steps of the current step, if any. Child steps have the same properties that of a step. | + +#### Include test steps only + +By default, the generated report includes steps and their child steps of all [Playwright\'s built-in test step categories](https://playwright.dev/docs/api/class-teststep#test-step-category), i.e. `hook`, `expect`, `pw:api` and `test.step`. + +For any reasons, if you want to include only the steps belonging to 'test.step' category, then set `testStepsOnly: true` in the configuration. + +You can refer to some [examples here](/docs/test-steps-only.md) to understand the impact of `testStepsOnly` option. + +> If you are **not** using test.step() in your tests, or using BDD styled tests with 'Background' keyword, then most likely you will need to include all steps in the report. Consider using the default behavior in such scenarios. + ### Annotations By setting `annotations: true` you can include annotations in the test extra property. @@ -209,10 +248,10 @@ export const test = _test.extend<{ _autoAttachMetadata: void }>({ }) // --------------------------------------------------------- - await use(/** our test doesn't need this fixture direcly */); + await use(/** our test doesn't need this fixture directly */); // --------------------------------------------------------- - // AFTER: There's nothing to cleanup in this fixutre + // AFTER: There's nothing to cleanup in this fixture }, { auto: true }], }) @@ -240,7 +279,7 @@ CTRF is a universal JSON test report schema that addresses the lack of a standar **Language and Framework Agnostic:** It provides a universal reporting schema that works seamlessly with any programming language and testing framework. -**Facilitates Better Analysis:** With a standardized format, programatically analyzing test outcomes across multiple platforms becomes more straightforward. +**Facilitates Better Analysis:** With a standardized format, programmatically analyzing test outcomes across multiple platforms becomes more straightforward. ## Support Us diff --git a/docs/test-steps-only-examples.md b/docs/test-steps-only-examples.md new file mode 100644 index 0000000..bf9b32d --- /dev/null +++ b/docs/test-steps-only-examples.md @@ -0,0 +1,927 @@ +# testStepsOnly Configuration Option - Examples + +You can control the kind of test steps included in report. + +If you need full verbosity or need to have results for your test hooks, then you will need to include all the steps. However, if all you need is either test result or top level steps without any child steps like fixtures, Playwright API calls etc., then you will need to opt for inclusion of only `test.step` category steps. + +You can control this behavior through `testStepsOnly` configuration option. + +By default, `testStepsOnly` is set to false and therefore the report includes all the steps, that you would see on the Playwright's default HTML report. This brings the `Playwright CTRF JSON Report` to parity with `Playwright's default HTML report`, in terms of test steps reporting. + +Below are some examples of how the report will look based on the `testStepsOnly` configuration. + +## `testStepsOnly: false` (default) + +### Simple test - No `test.step()` used. + +```typescript +test('has title @title', async ({ page }) => { + await page.goto('https://playwright.dev/') + + await expect(page).toHaveTitle(/Playwright/) +}) +``` + +A simple test like above, will be reported as: + +```json +{ + "name": "has title @title", + "status": "passed", + "duration": 646, + "start": 1729438996, + "stop": 1729438997, + "rawStatus": "passed", + "tags": [ + "@title" + ], + "type": "e2e", + "filePath": "/path-to-project-dir/tests/example.spec.ts", + "retries": 0, + "flaky": false, + "steps": [ + { + "name": "Before Hooks", + "status": "passed", + "extra": { + "category": "hook", + "duration": 139, + "childSteps": [ + { + "name": "fixture: browser", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 105, + "childSteps": [ + { + "name": "browserType.launch", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 104, + "childSteps": [] + } + } + ] + } + }, + { + "name": "fixture: context", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 6, + "childSteps": [ + { + "name": "browser.newContext", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 4, + "childSteps": [] + } + } + ] + } + }, + { + "name": "fixture: page", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 22, + "childSteps": [ + { + "name": "browserContext.newPage", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 22, + "childSteps": [] + } + } + ] + } + } + ] + } + }, + { + "name": "page.goto(https://playwright.dev/)", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 494, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 4, + "column": 14 + }, + "childSteps": [] + } + }, + { + "name": "expect.soft.toHaveTitle", + "status": "passed", + "extra": { + "category": "expect", + "duration": 46, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 7, + "column": 27 + }, + "childSteps": [] + } + }, + { + "name": "After Hooks", + "status": "passed", + "extra": { + "category": "hook", + "duration": 73, + "childSteps": [ + { + "name": "fixture: page", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 1, + "childSteps": [] + } + }, + { + "name": "fixture: context", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 0, + "childSteps": [] + } + } + ] + } + } + ], + "suite": "chromium > example.spec.ts", + "extra": { + "annotations": [] + } +}, +``` + +### Test with nested steps - `test.step()` usage + +```ts +// beforeEach hook, will be included in CTRF report. +test.beforeEach(async ({ page }) => { + await page.goto('https://playwright.dev/') +}) + +test( + 'with nested steps', + { tag: ['@steps', '@nested-steps'] }, + async ({ page }) => { + await test.step('Verify page title contains Playwright', async () => { + await expect.soft(page).toHaveTitle(/Playwright/) + }) + + // No test.step() usage for this step, but it will be included in CTRF report. + await page.getByRole('link', { name: 'Get started' }).click() + + await test.step('Verify Installation heading is visible', async () => { + await expect + .soft(page.getByRole('heading', { name: 'Installation' })) + .toBeVisible() + }) + } +) +``` + +A test having `test.step()` as steps, will be reported as: + +```json +{ + "name": "with nested steps", + "status": "passed", + "duration": 1050, + "start": 1729445458, + "stop": 1729445459, + "rawStatus": "passed", + "tags": [], + "type": "e2e", + "filePath": "/path-to-project-dir/tests/example.spec.ts", + "retries": 0, + "flaky": false, + "steps": [ + { + "name": "Before Hooks", + "status": "passed", + "extra": { + "category": "hook", + "duration": 775, + "childSteps": [ + { + "name": "beforeEach hook", + "status": "passed", + "extra": { + "category": "hook", + "duration": 774, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 3, + "column": 6 + }, + "childSteps": [ + { + "name": "fixture: browser", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 117, + "childSteps": [ + { + "name": "browserType.launch", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 116, + "childSteps": [] + } + } + ] + } + }, + { + "name": "fixture: context", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 8, + "childSteps": [ + { + "name": "browser.newContext", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 5, + "childSteps": [] + } + } + ] + } + }, + { + "name": "fixture: page", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 23, + "childSteps": [ + { + "name": "browserContext.newPage", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 22, + "childSteps": [] + } + } + ] + } + }, + { + "name": "page.goto(https://playwright.dev/)", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 620, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 4, + "column": 14 + }, + "childSteps": [] + } + } + ] + } + } + ] + } + }, + { + "name": "Verify page title contains Playwright", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 48, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 29, + "column": 14 + }, + "childSteps": [ + { + "name": "expect.soft.toHaveTitle", + "status": "passed", + "extra": { + "category": "expect", + "duration": 45, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 30, + "column": 29 + }, + "childSteps": [] + } + } + ] + } + }, + { + "name": "locator.getByRole('link', { name: 'Get started' }).click", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 80, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 34, + "column": 57 + }, + "childSteps": [] + } + }, + { + "name": "Verify Installation heading is visible", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 156, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 37, + "column": 14 + }, + "childSteps": [ + { + "name": "expect.soft.toBeVisible", + "status": "passed", + "extra": { + "category": "expect", + "duration": 155, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 38, + "column": 76 + }, + "childSteps": [] + } + } + ] + } + }, + { + "name": "After Hooks", + "status": "passed", + "extra": { + "category": "hook", + "duration": 110, + "childSteps": [ + { + "name": "fixture: page", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 0, + "childSteps": [] + } + }, + { + "name": "fixture: context", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 0, + "childSteps": [] + } + } + ] + } + } + ], + "suite": "chromium > example.spec.ts", + "extra": { + "annotations": [] + } +}, +} +``` + +### BDD Style test - using [Playwright-bdd](https://vitalets.github.io/playwright-bdd/#/) + +```gherkin +@login +Feature: User Login + + Background: + Given the User is on login page + + @locked_out_user + Scenario: Test that a Locked out user is not able to login despite using valid login credentials + When the User tries to login with "locked_out_user" as username and "secret_sauce" as password + Then the User should see a locked out error message +``` + +A BDD styled test like the one above, will be reported as: + +```json +{ + "name": "Test that a Locked out user is not able to login despite using valid login credentials", + "status": "passed", + "duration": 663, + "start": 1729441890, + "stop": 1729441891, + "rawStatus": "passed", + "tags": [], + "type": "e2e", + "filePath": "/path-to-project-dir/.features-gen/feature/login.feature.spec.js", + "retries": 0, + "flaky": false, + "steps": [ + { + "name": "Before Hooks", + "status": "passed", + "extra": { + "category": "hook", + "duration": 506, + "childSteps": [ + { + "name": "beforeEach hook", + "status": "passed", + "extra": { + "category": "hook", + "duration": 505, + "location": { + "file": "/path-to-project-dir/.features-gen/feature/login.feature.spec.js", + "line": 6, + "column": 8 + }, + "childSteps": [ + { + "name": "fixture: context", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 3, + "childSteps": [ + { + "name": "browser.newContext", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 1, + "childSteps": [] + } + } + ] + } + }, + { + "name": "fixture: page", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 16, + "childSteps": [ + { + "name": "browserContext.newPage", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 16, + "childSteps": [] + } + } + ] + } + }, + { + "name": "fixture: loginPage", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 0, + "location": { + "file": "/path-to-project-dir/pages/fixtures.ts", + "line": 16, + "column": 26 + }, + "childSteps": [] + } + }, + { + "name": "Given the User is on login page", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 484, + "location": { + "file": "/path-to-project-dir/.features-gen/feature/login.feature.spec.js", + "line": 7, + "column": 11 + }, + "childSteps": [ + { + "name": "page.goto(https://www.saucedemo.com)", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 483, + "location": { + "file": "/path-to-project-dir/pages/login-page.ts", + "line": 19, + "column": 25, + "function": "LoginPage.goto" + }, + "childSteps": [] + } + } + ] + } + } + ] + } + } + ] + } + }, + { + "name": "When the User tries to login with \"locked_out_user\" as username and \"secret_sauce\" as password", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 88, + "location": { + "file": "/path-to-project-dir/.features-gen/feature/login.feature.spec.js", + "line": 49, + "column": 11 + }, + "childSteps": [ + { + "name": "locator.fill([data-test=\"username\"])", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 21, + "location": { + "file": "/path-to-project-dir/pages/login-page.ts", + "line": 23, + "column": 29, + "function": "LoginPage.doLogin" + }, + "childSteps": [] + } + }, + { + "name": "locator.fill([data-test=\"password\"])", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 7, + "location": { + "file": "/path-to-project-dir/pages/login-page.ts", + "line": 24, + "column": 29, + "function": "LoginPage.doLogin" + }, + "childSteps": [] + } + }, + { + "name": "locator.click([data-test=\"login-button\"])", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 57, + "location": { + "file": "/path-to-project-dir/pages/login-page.ts", + "line": 25, + "column": 32, + "function": "LoginPage.doLogin" + }, + "childSteps": [] + } + } + ] + } + }, + { + "name": "Then the User should see a locked out error message", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 8, + "location": { + "file": "/path-to-project-dir/.features-gen/feature/login.feature.spec.js", + "line": 50, + "column": 11 + }, + "childSteps": [ + { + "name": "expect.toBeVisible", + "status": "passed", + "extra": { + "category": "expect", + "duration": 3, + "location": { + "file": "/path-to-project-dir/step-definitions/login.steps.ts", + "line": 13, + "column": 42, + "function": "Object." + }, + "childSteps": [] + } + }, + { + "name": "locator.textContent([data-test=\"error\"])", + "status": "passed", + "extra": { + "category": "pw:api", + "duration": 2, + "location": { + "file": "/path-to-project-dir/step-definitions/login.steps.ts", + "line": 15, + "column": 52, + "function": "Object." + }, + "childSteps": [] + } + }, + { + "name": "expect.toEqual", + "status": "passed", + "extra": { + "category": "expect", + "duration": 1, + "location": { + "file": "/path-to-project-dir/step-definitions/login.steps.ts", + "line": 16, + "column": 23, + "function": "Object." + }, + "childSteps": [] + } + } + ] + } + }, + { + "name": "After Hooks", + "status": "passed", + "extra": { + "category": "hook", + "duration": 61, + "childSteps": [ + { + "name": "fixture: loginPage", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 0, + "location": { + "file": "/path-to-project-dir/pages/fixtures.ts", + "line": 16, + "column": 26 + }, + "childSteps": [] + } + }, + { + "name": "fixture: page", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 0, + "childSteps": [] + } + }, + { + "name": "fixture: context", + "status": "passed", + "extra": { + "category": "fixture", + "duration": 0, + "childSteps": [] + } + } + ] + } + } + ], + "suite": "chromium > feature/login.feature.spec.js > User Login", + "extra": { + "annotations": [] + } +} +``` + +## `testStepsOnly: true` + +### Simple test - No `test.step()` used. + +```ts +test('has title @title', async ({ page }) => { + await page.goto('https://playwright.dev/') + + await expect(page).toHaveTitle(/Playwright/) +}) +``` + +Simple test above will be reported as: + +> ⚠️ Notice that no steps are reported in CTRF JSON, as both the steps belong to 'pw:api' and 'expect' categories, respectively. + +```json +{ + "name": "has title @title", + "status": "passed", + "duration": 1140, + "start": 1729428643, + "stop": 1729428644, + "rawStatus": "passed", + "tags": [ + "@title" + ], + "type": "e2e", + "filePath": "/path-to-project-dir/tests/example.spec.ts", + "retries": 0, + "flaky": false, + "steps": [], + "suite": "chromium > example.spec.ts", + "extra": { + "annotations": [] + } +}, +``` + +### Test with nested steps - `test.step()` usage + +```ts +// beforeEach hook, will NOT be included in CTRF report. +test.beforeEach(async ({ page }) => { + await page.goto('https://playwright.dev/') +}) + +test( + 'with nested steps', + { tag: ['@steps', '@nested-steps'] }, + async ({ page }) => { + await test.step('Navigate to Playwright homepage', async () => { + await page.goto('https://playwright.dev/') + }) + + await test.step('Verify page title contains Playwright', async () => { + await expect(page).toHaveTitle(/Playwright/) + }) + + // No test.step() used for this step, it will NOT be included in CTRF report. + await page.getByRole('link', { name: 'Get started' }).click() + + await test.step('Verify Installation heading is visible', async () => { + await expect( + page.getByRole('heading', { name: 'Installation' }) + ).toBeVisible() + }) + } +) +``` + +A test with `test.step()` usage, will be reported as: + +> ⚠️ Notice that the step inside 'beforeEach' hook is not included as it is a child step of 'beforeEach' hook (which belongs to 'hook' category), and the step where 'test.step()' is not used inside the test, is also omitted because it belongs to 'pw:api' category. Similarly, all steps inside the 'beforeAll/afterAll/afterEach' hooks will not be included. + +```json +{ + "name": "with nested steps", + "status": "passed", + "duration": 917, + "start": 1729442874, + "stop": 1729442875, + "rawStatus": "passed", + "tags": [], + "type": "e2e", + "filePath": "/path-to-project-dir/tests/example.spec.ts", + "retries": 0, + "flaky": false, + "steps": [ + { + "name": "Verify page title contains Playwright", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 45, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 29, + "column": 14 + }, + "childSteps": [] + } + }, + { + "name": "Verify Installation heading is visible", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 363, + "location": { + "file": "/path-to-project-dir/tests/example.spec.ts", + "line": 37, + "column": 14 + }, + "childSteps": [] + } + } + ], + "suite": "chromium > example.spec.ts", + "extra": { + "annotations": [] + } +} +``` + +### BDD Style test - using [Playwright-bdd](https://vitalets.github.io/playwright-bdd/#/) + +```gherkin +@login +Feature: User Login + + Background: + Given the User is on login page + + @locked_out_user + Scenario: Test that a Locked out user is not able to login despite using valid login credentials + When the User tries to login with "locked_out_user" as username and "secret_sauce" as password + Then the User should see a locked out error message +``` + +A BDD styled test like the one above, is reported as shown below. + +> ⚠️ Notice that the step in the 'Background' hook is not included. Similarly, any steps or fixtures executed as part of 'After' hooks will be omitted as well. + +```json +{ + "name": "Test that a Locked out user is not able to login despite using valid login credentials", + "status": "passed", + "duration": 644, + "start": 1729431624, + "stop": 1729431625, + "rawStatus": "passed", + "tags": [], + "type": "e2e", + "filePath": "/path-to-project-dir/.features-gen/feature/login.feature.spec.js", + "retries": 0, + "flaky": false, + "steps": [ + { + "name": "When the User tries to login with \"locked_out_user\" as username and \"secret_sauce\" as password", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 69, + "location": { + "file": "/path-to-project-dir/.features-gen/feature/login.feature.spec.js", + "line": 49, + "column": 11 + }, + "childSteps": [] + } + }, + { + "name": "Then the User should see a locked out error message", + "status": "passed", + "extra": { + "category": "test.step", + "duration": 7, + "location": { + "file": "/path-to-project-dir/.features-gen/feature/login.feature.spec.js", + "line": 50, + "column": 11 + }, + "childSteps": [] + } + } + ], + "suite": "chromium > feature/login.feature.spec.js > User Login", + "extra": { + "annotations": [] + } +} +``` + +## BDD styled tests with [Cucumber](https://cucumber.io/docs/guides/overview/) + +BDD styled tests that use cucumber as test runner are not supported. Only [playwright-bdd](https://vitalets.github.io/playwright-bdd/#/) is supported because it uses [Playwright Test](https://playwright.dev/docs/test-configuration) as a test runner to execute the tests. diff --git a/src/generate-report.ts b/src/generate-report.ts index b14c7db..91347cc 100644 --- a/src/generate-report.ts +++ b/src/generate-report.ts @@ -15,12 +15,14 @@ import { type CtrfReport, type CtrfTest, type CtrfEnvironment, + type Step, } from '../types/ctrf' interface ReporterConfigOptions { outputFile?: string outputDir?: string minimal?: boolean + testStepsOnly?: boolean screenshot?: boolean annotations?: boolean testType?: string @@ -53,6 +55,7 @@ class GenerateCtrfReport implements Reporter { outputFile: config?.outputFile ?? this.defaultOutputFile, outputDir: config?.outputDir ?? this.defaultOutputDir, minimal: config?.minimal ?? false, + testStepsOnly: config?.testStepsOnly ?? false, screenshot: config?.screenshot ?? false, annotations: config?.annotations ?? false, testType: config?.testType ?? 'e2e', @@ -403,15 +406,26 @@ class GenerateCtrfReport implements Reporter { } processStep(test: CtrfTest, step: TestStep): void { - if (step.category === 'test.step') { - const stepStatus = - step.error === undefined - ? this.mapPlaywrightStatusToCtrf('passed') - : this.mapPlaywrightStatusToCtrf('failed') - const currentStep = { - name: step.title, - status: stepStatus, + const stepStatus = + step.error === undefined + ? this.mapPlaywrightStatusToCtrf('passed') + : this.mapPlaywrightStatusToCtrf('failed') + const currentStep = { + name: step.title, + status: stepStatus, + extra: { + category: step.category, + duration: step.duration, + location: step.location, + childSteps: [], + }, + } + + if (this.reporterConfigOptions.testStepsOnly ?? false) { + if (step.category === 'test.step') { + test.steps?.push(currentStep) } + } else { test.steps?.push(currentStep) } @@ -419,7 +433,39 @@ class GenerateCtrfReport implements Reporter { if (childSteps.length > 0) { childSteps.forEach((cStep) => { - this.processStep(test, cStep) + this.processChildSteps(cStep, currentStep) + }) + } + } + + processChildSteps(step: TestStep, parentStep: Step): void { + const stepStatus = + step.error === undefined + ? this.mapPlaywrightStatusToCtrf('passed') + : this.mapPlaywrightStatusToCtrf('failed') + const currentStep = { + name: step.title, + status: stepStatus, + extra: { + category: step.category, + duration: step.duration, + location: step.location, + childSteps: [], + }, + } + + if (this.reporterConfigOptions.testStepsOnly ?? false) { + if (step.category === 'test.step') { + parentStep.extra?.childSteps.push(currentStep) + } + } else { + parentStep.extra?.childSteps.push(currentStep) + } + + if (step.steps.length > 0) { + const childSteps = step.steps + childSteps.forEach((cStep) => { + this.processChildSteps(cStep, currentStep) }) } } diff --git a/types/ctrf.d.ts b/types/ctrf.d.ts index 61a72c6..35624be 100644 --- a/types/ctrf.d.ts +++ b/types/ctrf.d.ts @@ -72,6 +72,7 @@ export interface Tool { export interface Step { name: string status: CtrfTestState + extra?: Record } export type CtrfTestState =