chore: inject resources with CDP (#7186)

* proxyless: inject resources using cdp

* fix tests

* small refactoring

* small refactoring

* update hammerhead

* stub proxyless tests

* fix task script creation

* fix proxyless run

* revert refactoring

Author: miherlosev

Gulpfile.js

@@ -36,6 +36,7 @@ const {
     MULTIPLE_WINDOWS_TESTS_GLOB,
     DEBUG_GLOB_1,
     DEBUG_GLOB_2,
+    PROXYLESS_TESTS_GLOB,
 } = require('./gulp/constants/functional-test-globs');
 
 const {
@@ -424,7 +425,7 @@ gulp.task('test-functional-local-debug-1', gulp.series('prepare-tests', 'test-fu
 gulp.task('test-functional-local-debug-2', gulp.series('prepare-tests', 'test-functional-local-debug-run-2'));
 
 gulp.step('test-functional-local-proxyless-run', () => {
-    return testFunctional(TESTS_GLOB, functionalTestConfig.testingEnvironmentNames.localHeadlessChrome, { isProxyless: true });
+    return testFunctional(PROXYLESS_TESTS_GLOB, functionalTestConfig.testingEnvironmentNames.localHeadlessChrome, { isProxyless: true });
 });
 
 gulp.task('test-functional-local-proxyless', gulp.series('prepare-tests', 'test-functional-local-proxyless-run'));

gulp/constants/functional-test-globs.js

@@ -27,6 +27,8 @@ const DEBUG_GLOB_2 = [
     ...SCREENSHOT_TESTS_GLOB.map(glob => `!${glob}`),
 ];
 
+const PROXYLESS_TESTS_GLOB = [];
+
 module.exports = {
     TESTS_GLOB,
     LEGACY_TESTS_GLOB,
@@ -36,4 +38,5 @@ module.exports = {
     DEBUG_GLOB_1,
     DEBUG_GLOB_2,
     SCREENSHOT_TESTS_GLOB,
+    PROXYLESS_TESTS_GLOB,
 };

package.json

@@ -137,7 +137,7 @@
     "source-map-support": "^0.5.16",
     "strip-bom": "^2.0.0",
     "testcafe-browser-tools": "2.0.23",
-    "testcafe-hammerhead": "24.5.24",
+    "testcafe-hammerhead": "24.6.0",
     "testcafe-legacy-api": "5.1.4",
     "testcafe-reporter-dashboard": "1.0.0-rc.3",
     "testcafe-reporter-json": "^2.1.0",

src/browser/connection/browser-connection-tracker.ts

@@ -0,0 +1,19 @@
+import BrowserConnection from './index';
+
+class BrowserConnectionTracker {
+    public activeBrowserConnections: { [id: string]: BrowserConnection };
+
+    public constructor () {
+        this.activeBrowserConnections = {};
+    }
+
+    public add (connection: BrowserConnection): void {
+        this.activeBrowserConnections[connection.id] = connection;
+    }
+
+    public remove (connection: BrowserConnection): void {
+        delete this.activeBrowserConnections[connection.id];
+    }
+}
+
+export default new BrowserConnectionTracker();

src/browser/connection/gateway.ts

@@ -18,16 +18,15 @@ import SERVICE_ROUTES from './service-routes';
 export default class BrowserConnectionGateway {
     private _connections: Dictionary<BrowserConnection> = {};
     private _remotesQueue: RemotesQueue;
-    public readonly domain: string;
     public readonly connectUrl: string;
     public retryTestPages: boolean;
+    public readonly proxy: Proxy;
 
     public constructor (proxy: Proxy, options: { retryTestPages: boolean }) {
         this._remotesQueue   = new RemotesQueue();
-        // @ts-ignore Need to improve typings of the 'testcafe-hammerhead' module
-        this.domain          = (proxy as any).server1Info.domain;
-        this.connectUrl      = `${this.domain}/browser/connect`;
+        this.connectUrl      = proxy.resolveRelativeServiceUrl('/browser/connect');
         this.retryTestPages  = options.retryTestPages;
+        this.proxy           = proxy;
 
         this._registerRoutes(proxy);
     }

src/browser/connection/index.ts

@@ -17,7 +17,6 @@ import BrowserConnectionStatus from './status';
 import HeartbeatStatus from './heartbeat-status';
 import { GeneralError, TimeoutError } from '../../errors/runtime';
 import { RUNTIME_ERRORS } from '../../errors/types';
-import { Dictionary } from '../../configuration/interfaces';
 import BrowserConnectionGateway from './gateway';
 import BrowserJob from '../../runner/browser-job';
 import WarningLog from '../../notifications/warning-log';
@@ -32,11 +31,16 @@ import {
     REMOTE_BROWSER_INIT_TIMEOUT,
 } from '../../utils/browser-connection-timeouts';
 import MessageBus from '../../utils/message-bus';
+import BrowserConnectionTracker from './browser-connection-tracker';
+import TestRun from '../../test-run';
+// @ts-ignore
+import { TestRun as LegacyTestRun } from 'testcafe-legacy-api';
+import { Proxy } from 'testcafe-hammerhead';
 
 const getBrowserConnectionDebugScope = (id: string): string => `testcafe:browser:connection:${id}`;
 
-const IDLE_PAGE_TEMPLATE                         = read('../../client/browser/idle-page/index.html.mustache');
-const connections: Dictionary<BrowserConnection> = {};
+const IDLE_PAGE_TEMPLATE = read('../../client/browser/idle-page/index.html.mustache');
+
 
 interface DisconnectionPromise<T> extends Promise<T> {
     resolve: Function;
@@ -83,31 +87,31 @@ export default class BrowserConnection extends EventEmitter {
     public permanent: boolean;
     public previousActiveWindowId: string | null;
     private readonly disableMultipleWindows: boolean;
-    private readonly proxyless: boolean;
+    public readonly proxyless: boolean;
     private readonly HEARTBEAT_TIMEOUT: number;
     private readonly BROWSER_CLOSE_TIMEOUT: number;
     private readonly BROWSER_RESTART_TIMEOUT: number;
     public readonly id: string;
     private readonly jobQueue: BrowserJob[];
     private readonly initScriptsQueue: InitScriptTask[];
-    private browserConnectionGateway: BrowserConnectionGateway;
+    public browserConnectionGateway: BrowserConnectionGateway;
     private disconnectionPromise: DisconnectionPromise<void> | null;
     private testRunAborted: boolean;
     public status: BrowserConnectionStatus;
     private heartbeatTimeout: NodeJS.Timeout | null;
     private pendingTestRunUrl: string | null;
-    public readonly url: string;
-    public readonly idleUrl: string;
-    private forcedIdleUrl: string;
-    private readonly initScriptUrl: string;
-    public readonly heartbeatRelativeUrl: string;
-    public readonly statusRelativeUrl: string;
-    public readonly statusDoneRelativeUrl: string;
-    private readonly heartbeatUrl: string;
-    private readonly statusUrl: string;
-    public readonly activeWindowIdUrl: string;
-    public readonly closeWindowUrl: string;
-    private statusDoneUrl: string;
+    public url = '';
+    public idleUrl = '';
+    private forcedIdleUrl = '';
+    private initScriptUrl = '';
+    public heartbeatUrl = '';
+    public statusUrl = '';
+    public activeWindowIdUrl = '';
+    public closeWindowUrl = '';
+    public statusDoneUrl = '';
+    public heartbeatRelativeUrl = '';
+    public statusRelativeUrl = '';
+    public statusDoneRelativeUrl = '';
     private readonly debugLogger: debug.Debugger;
     private osInfo: OSInfo | null = null;
 
@@ -155,24 +159,10 @@ export default class BrowserConnection extends EventEmitter {
         this.disableMultipleWindows = disableMultipleWindows;
         this.proxyless              = proxyless;
 
-        this.url           = `${gateway.domain}/browser/connect/${this.id}`;
-        this.idleUrl       = `${gateway.domain}/browser/idle/${this.id}`;
-        this.forcedIdleUrl = `${gateway.domain}/browser/idle-forced/${this.id}`;
-        this.initScriptUrl = `${gateway.domain}/browser/init-script/${this.id}`;
-
-        this.heartbeatRelativeUrl  = `/browser/heartbeat/${this.id}`;
-        this.statusRelativeUrl     = `/browser/status/${this.id}`;
-        this.statusDoneRelativeUrl = `/browser/status-done/${this.id}`;
-        this.activeWindowIdUrl     = `/browser/active-window-id/${this.id}`;
-        this.closeWindowUrl        = `/browser/close-window/${this.id}`;
-
-        this.heartbeatUrl  = `${gateway.domain}${this.heartbeatRelativeUrl}`;
-        this.statusUrl     = `${gateway.domain}${this.statusRelativeUrl}`;
-        this.statusDoneUrl = `${gateway.domain}${this.statusDoneRelativeUrl}`;
-
+        this._buildCommunicationUrls(gateway.proxy);
         this._setEventHandlers();
 
-        connections[this.id] = this;
+        BrowserConnectionTracker.add(this);
 
         this.previousActiveWindowId = null;
 
@@ -182,6 +172,24 @@ export default class BrowserConnection extends EventEmitter {
         process.nextTick(() => this._runBrowser());
     }
 
+    private _buildCommunicationUrls (proxy: Proxy): void {
+        this.url               = proxy.resolveRelativeServiceUrl(`/browser/connect/${this.id}`);
+        this.idleUrl           = proxy.resolveRelativeServiceUrl(`/browser/idle/${this.id}`);
+        this.forcedIdleUrl     = proxy.resolveRelativeServiceUrl(`/browser/idle-forced/${this.id}`);
+        this.initScriptUrl     = proxy.resolveRelativeServiceUrl(`/browser/init-script/${this.id}`);
+
+        this.heartbeatRelativeUrl  = `/browser/heartbeat/${this.id}`;
+        this.statusRelativeUrl     = `/browser/status/${this.id}`;
+        this.statusDoneRelativeUrl = `/browser/status-done/${this.id}`;
+        this.activeWindowIdUrl     = `/browser/active-window-id/${this.id}`;
+        this.closeWindowUrl        = `/browser/close-window/${this.id}`;
+
+        this.heartbeatUrl      = proxy.resolveRelativeServiceUrl(this.heartbeatRelativeUrl);
+        this.statusUrl         = proxy.resolveRelativeServiceUrl(this.statusRelativeUrl);
+        this.statusDoneUrl     = proxy.resolveRelativeServiceUrl(this.statusDoneRelativeUrl);
+
+    }
+
     public set messageBus (messageBus: MessageBus) {
         this._messageBus         = messageBus;
         this.warningLog.callback = WarningLog.createAddWarningCallback(this._messageBus);
@@ -278,8 +286,12 @@ export default class BrowserConnection extends EventEmitter {
         return this.hasQueuedJobs ? await this.currentJob.popNextTestRunUrl(this) : null;
     }
 
+    public getCurrentTestRun (): LegacyTestRun | TestRun | null {
+        return this.currentJob ? this.currentJob.currentTestRun : null;
+    }
+
     public static getById (id: string): BrowserConnection | null {
-        return connections[id] || null;
+        return BrowserConnectionTracker.activeBrowserConnections[id] || null;
     }
 
     private async _restartBrowser (): Promise<void> {
@@ -454,7 +466,7 @@ export default class BrowserConnection extends EventEmitter {
         if (this.heartbeatTimeout)
             clearTimeout(this.heartbeatTimeout);
 
-        delete connections[this.id];
+        BrowserConnectionTracker.remove(this);
 
         this.status = BrowserConnectionStatus.closed;
         this.emit(BrowserConnectionStatus.closed);

src/browser/provider/built-in/dedicated/chrome/cdp-client/index.ts

@@ -49,16 +49,14 @@ interface VideoFrameData {
 export class BrowserClient {
     private _clients: Dictionary<ProtocolApiInfo> = {};
     private readonly _runtimeInfo: RuntimeInfo;
-    private readonly _proxyless: boolean;
     private _parentTarget?: remoteChrome.TargetInfo;
     private readonly debugLogger: debug.Debugger;
     private readonly _videoFramesBuffer: VideoFrameData[];
     private _lastFrame: VideoFrameData | null;
 
-    public constructor (runtimeInfo: RuntimeInfo, proxyless: boolean) {
+    public constructor (runtimeInfo: RuntimeInfo) {
         this._runtimeInfo = runtimeInfo;
         this.debugLogger  = debug(DEBUG_SCOPE(runtimeInfo.browserId));
-        this._proxyless   = proxyless;
 
         runtimeInfo.browserClient = this;
 

src/browser/provider/built-in/dedicated/chrome/index.js

@@ -10,6 +10,7 @@ import {
 } from './local-chrome';
 import { GET_WINDOW_DIMENSIONS_INFO_SCRIPT } from '../../../utils/client-functions';
 import { BrowserClient } from './cdp-client';
+import RequestsInterceptor from './requests-interceptor';
 
 const MIN_AVAILABLE_DIMENSION = 50;
 
@@ -43,6 +44,13 @@ export default {
         this.setUserAgentMetaInfo(browserId, metaInfo, options);
     },
 
+    async _setupProxyless (browserId, browserClient) {
+        const requestsInterceptor = new RequestsInterceptor(browserId);
+        const cdpClient           = await browserClient.getActiveClient();
+
+        await requestsInterceptor.setup(cdpClient);
+    },
+
     async openBrowser (browserId, pageUrl, config, disableMultipleWindows, proxyless) {
         const parsedPageUrl = parseUrl(pageUrl);
         const runtimeInfo   = await this._createRunTimeInfo(parsedPageUrl.hostname, config, disableMultipleWindows);
@@ -70,7 +78,7 @@ export default {
         if (!disableMultipleWindows)
             runtimeInfo.activeWindowId = this.calculateWindowId();
 
-        const browserClient = new BrowserClient(runtimeInfo, proxyless);
+        const browserClient = new BrowserClient(runtimeInfo);
 
         this.openedBrowsers[browserId] = runtimeInfo;
 
@@ -79,6 +87,9 @@ export default {
         await this._ensureWindowIsExpanded(browserId, runtimeInfo.viewportSize);
 
         this._setUserAgentMetaInfoForEmulatingDevice(browserId, runtimeInfo.config);
+
+        if (proxyless)
+            await this._setupProxyless(browserId, browserClient);
     },
 
     async closeBrowser (browserId, closingInfo = {}) {

src/browser/provider/built-in/dedicated/chrome/requests-interceptor.ts

@@ -0,0 +1,89 @@
+import { ProtocolApi } from 'chrome-remote-interface';
+import Protocol from 'devtools-protocol';
+import RequestPausedEvent = Protocol.Fetch.RequestPausedEvent;
+import RequestPattern = Protocol.Fetch.RequestPattern;
+import GetResponseBodyResponse = Protocol.Fetch.GetResponseBodyResponse;
+import {
+    injectResources,
+    PageInjectableResources,
+    INJECTABLE_SCRIPTS as HAMMERHEAD_INJECTABLE_SCRIPTS,
+} from 'testcafe-hammerhead';
+import BrowserConnection from '../../../../connection';
+import { SCRIPTS, TESTCAFE_UI_STYLES } from '../../../../../assets/injectables';
+
+const HTTP_STATUS_OK = 200;
+
+export default class RequestsInterceptor {
+    private readonly _browserId: string;
+
+    public constructor (browserId: string) {
+        this._browserId = browserId;
+    }
+
+    private _getResponseAsString (response: GetResponseBodyResponse): string {
+        return response.base64Encoded
+            ? Buffer.from(response.body, 'base64').toString()
+            : response.body;
+    }
+
+    private async _prepareInjectableResources (): Promise<PageInjectableResources> {
+        const browserConnection = BrowserConnection.getById(this._browserId) as BrowserConnection;
+        const proxy             = browserConnection.browserConnectionGateway.proxy;
+        const windowId          = browserConnection.activeWindowId;
+
+        const taskScript = await browserConnection.currentJob.currentTestRun.session.getTaskScript({
+            referer:     '',
+            cookieUrl:   '',
+            isIframe:    false,
+            withPayload: true,
+            serverInfo:  proxy.server1Info,
+            windowId,
+        });
+
+        const injectableResources = {
+            stylesheets: [
+                TESTCAFE_UI_STYLES,
+            ],
+            scripts: [
+                ...HAMMERHEAD_INJECTABLE_SCRIPTS,
+                ...SCRIPTS,
+            ],
+            embeddedScripts: [taskScript],
+        };
+
+        injectableResources.scripts     = injectableResources.scripts.map(script => proxy.resolveRelativeServiceUrl(script));
+        injectableResources.stylesheets = injectableResources.stylesheets.map(style => proxy.resolveRelativeServiceUrl(style));
+
+        return injectableResources;
+    }
+
+    public async setup (client: ProtocolApi): Promise<void> {
+        const fetchAllDocumentsPattern = {
+            urlPattern:   '*',
+            resourceType: 'Document',
+            requestStage: 'Response',
+        } as RequestPattern;
+
+        await client.Fetch.enable({ patterns: [fetchAllDocumentsPattern] });
+
+        client.Fetch.on('requestPaused', async (params: RequestPausedEvent) => {
+            const {
+                requestId,
+                responseHeaders,
+                responseStatusCode,
+            } = params;
+
+            const responseObj         = await client.Fetch.getResponseBody({ requestId });
+            const responseStr         = this._getResponseAsString(responseObj);
+            const injectableResources = await this._prepareInjectableResources();
+            const updatedResponseStr  = injectResources(responseStr, injectableResources);
+
+            await client.Fetch.fulfillRequest({
+                requestId,
+                responseCode:    responseStatusCode || HTTP_STATUS_OK,
+                responseHeaders: responseHeaders || [],
+                body:            Buffer.from(updatedResponseStr).toString('base64'),
+            });
+        });
+    }
+}

src/client/test-run/index.js.mustache

@@ -26,6 +26,7 @@
     var fixtureName                = {{{fixtureName}}};
     var testName                   = {{{testName}}};
     var canUseDefaultWindowActions = {{{canUseDefaultWindowActions}}};
+    var proxyless                  = {{{proxyless}}};
 
     var ClientDriver = window['%testCafeDriver%'];
     var driver       = new ClientDriver(testRunId,
@@ -39,7 +40,8 @@
             dialogHandler:              dialogHandler,
             retryTestPages:             retryTestPages,
             speed:                      speed,
-            canUseDefaultWindowActions: canUseDefaultWindowActions
+            canUseDefaultWindowActions: canUseDefaultWindowActions,
+            proxyless:                  proxyless
         }
     );