testing: Add Theia Playwright framework (#10337)

Co-authored-by: Nina Doschek <[email protected]>
Co-authored-by: Johannes Faltermeier <[email protected]>
Co-authored-by: Martin Fleck <[email protected]>
Co-authored-by: Simon Graband <[email protected]>
Co-authored-by: Tobias Ortmayr <[email protected]>

Fixes #10337

Author: planger

.vscode/launch.json

@@ -176,6 +176,21 @@
       "outFiles": [
         "${workspaceFolder}/../.js"
       ]
+    },
+    {
+      "name": "Debug selected system test file with Playwright",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/examples/playwright/node_modules/.bin/playwright",
+      "args": [
+        "test",
+        "--debug",
+        "--config=./configs/playwright.debug.config.ts",
+        "${fileBasenameNoExtension}"
+      ],
+      "cwd": "${workspaceFolder}/examples/playwright",
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen"
     }
   ],
   "compounds": [

examples/playwright/.eslintrc.js

@@ -0,0 +1,12 @@
+/** @type {import('eslint').Linter.Config} */
+module.exports = {
+    extends: [
+        '../../configs/build.eslintrc.json',
+        './configs/ui-tests.eslintrc.json',
+        './configs/ui-tests.playwright.eslintrc.json'
+    ],
+    parserOptions: {
+        tsconfigRootDir: __dirname,
+        project: 'tsconfig.json'
+    }
+};

examples/playwright/.gitignore

@@ -0,0 +1,2 @@
+allure-results
+test-results

examples/playwright/README.md

@@ -0,0 +1,28 @@
+# Theia 🎭 Playwright: A System Testing Framework for Theia
+
+Theia 🎭 Playwright is a [page object](https://martinfowler.com/bliki/PageObject.html) framework based on [Playwright](https://github.com/microsoft/playwright) for developing system tests of [Theia](https://github.com/eclipse-theia/theia)-based applications. See it in action below.
+
+<div style='margin:0 auto;width:70%;'>
+
+![Theia System Testing in Action](./docs/images/teaser.gif)
+
+</div>
+
+The Theia 🎭 Playwright page objects introduce abstraction over Theia's user interfaces, encapsulating the details of the user interface interactions, wait conditions, etc., to help keeping your tests more concise, maintainable, and stable.
+Ready for an [example](./docs/GETTING_STARTED.md)?
+
+The actual interaction with the Theia application is implemented with 🎭 Playwright in Typescript. Thus, we can take advantage of [Playwright's benefits](https://playwright.dev/docs/why-playwright/) and run or debug tests headless or headful across all modern browsers.
+Check out [Playwright's documentation](https://playwright.dev/docs/intro) for more information.
+
+This page object framework not only covers Theia's generic capabilities, such as handling views, the quick command palette, file explorer etc.
+It is [extensible](./docs/EXTENSIBILITY.md) so you can add dedicated page objects for custom Theia components, such as custom views, editors, menus, etc.
+
+## Documentation
+
+-   [Getting Started](./docs/GETTING_STARTED.md)
+-   [Extensibility](./docs/EXTENSIBILITY.md)
+-   [Building and Developing Theia 🎭 Playwright](./docs/DEVELOPING.md)
+
+## Get in touch
+
+If you have problems, find bugs or have questions, feel free to get in contact via the bugs and discussions.

examples/playwright/configs/playwright.config.ts

@@ -0,0 +1,42 @@
+/********************************************************************************
+ * Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+    testDir: '../lib/tests',
+    testMatch: ['**/*.js'],
+    workers: 2,
+    // Timeout for each test in milliseconds.
+    timeout: 60 * 1000,
+    use: {
+        baseURL: 'http://localhost:3000',
+        browserName: 'chromium',
+        screenshot: 'only-on-failure',
+        viewport: { width: 1920, height: 1080 }
+    },
+    snapshotDir: '../tests/snapshots',
+    expect: {
+        toMatchSnapshot: { threshold: 0.15 }
+    },
+    preserveOutput: 'failures-only',
+    reporter: [
+        ['list'],
+        ['allure-playwright']
+    ]
+};
+
+export default config;

examples/playwright/configs/playwright.debug.config.ts

@@ -0,0 +1,27 @@
+/********************************************************************************
+ * Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { PlaywrightTestConfig } from '@playwright/test';
+
+import baseConfig from './playwright.config';
+
+const debugConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    timeout: 15000000
+};
+
+export default debugConfig;

examples/playwright/configs/playwright.headful.config.ts

@@ -0,0 +1,30 @@
+/********************************************************************************
+ * Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { PlaywrightTestConfig } from '@playwright/test';
+
+import baseConfig from './playwright.config';
+
+const headfulConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    use: {
+        ...baseConfig.use,
+        headless: false
+    }
+};
+
+export default headfulConfig;

examples/playwright/configs/ui-tests.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  // override existing rules for ui-tests package
+  "rules": {
+    "no-undef": "off", // disabled due to 'browser', '$', '$$'
+    "no-unused-expressions": "off"
+  }
+}

examples/playwright/configs/ui-tests.playwright.eslintrc.json

@@ -0,0 +1,6 @@
+{
+  // override existing rules for ui-tests playwright package
+  "rules": {
+    "no-null/no-null": "off"
+  }
+}

examples/playwright/docs/DEVELOPING.md

@@ -0,0 +1,26 @@
+# Building and developing Theia 🎭 Playwright
+
+## Building
+
+Run `yarn` in the root directory of the repository.
+
+## Executing the tests
+
+### Prerequisites
+
+To work with the tests the Theia Application under test needs to be running.
+
+Run `yarn browser start` to start the browser-app located in this repository.
+
+You may also use the `Launch Browser Backend` launch configuration in VS Code.
+
+### Running the tests headless
+
+To start the tests run `yarn ui-tests` in the root of this repository. This will start the tests located in `tests` in a headless mode.
+
+To only run a single test file, the path of a test file can be set with `yarn ui-tests <path-to-file>` or `yarn ui-tests -g "<partial test file name>"`.
+See the [Playwright Test command line documentation](https://playwright.dev/docs/intro#command-line).
+
+### Debugging the tests
+
+Please refer to the section [debugging tests](./GETTING_STARTED.md#debugging-the-tests).