From cc30804ca700a32d580614c6e1e182fbb9ed9bcb Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Sat, 19 Jul 2025 15:14:36 -0700 Subject: [PATCH 01/12] store frameID -> stagehandPage mapping --- lib/StagehandContext.ts | 16 ++++++++++++++++ lib/StagehandPage.ts | 31 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/lib/StagehandContext.ts b/lib/StagehandContext.ts index 41b6fe1fd..92062c3e9 100644 --- a/lib/StagehandContext.ts +++ b/lib/StagehandContext.ts @@ -12,6 +12,7 @@ export class StagehandContext { private readonly intContext: EnhancedContext; private pageMap: WeakMap; private activeStagehandPage: StagehandPage | null = null; + private readonly frameIdMap: Map = new Map(); private constructor(context: PlaywrightContext, stagehand: Stagehand) { this.stagehand = stagehand; @@ -101,6 +102,21 @@ export class StagehandContext { return instance; } + public get frameIdLookup(): ReadonlyMap { + return this.frameIdMap; + } + + public registerFrameId(frameId: string, page: StagehandPage): void { + this.frameIdMap.set(frameId, page); + } + + public unregisterFrameId(frameId: string): void { + this.frameIdMap.delete(frameId); + } + + public getStagehandPageByFrameId(frameId: string): StagehandPage | undefined { + return this.frameIdMap.get(frameId); + } public get context(): EnhancedContext { return this.intContext; diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts index 353631963..67bedb6df 100644 --- a/lib/StagehandPage.ts +++ b/lib/StagehandPage.ts @@ -56,6 +56,12 @@ export class StagehandPage { [undefined, 0], ]); + private rootFrameId!: string; + + public get frameId(): string { + return this.rootFrameId; + } + constructor( page: PlaywrightPage, stagehand: Stagehand, @@ -426,6 +432,31 @@ ${scriptContent} \ this.intPage = new Proxy(page, handler) as unknown as Page; this.initialized = true; + const { frameTree } = + await this.sendCDP( + "Page.getFrameTree", + ); + this.rootFrameId = frameTree.frame.id; + this.intContext.registerFrameId(this.rootFrameId, this); + + this.page.once("close", () => { + this.intContext.unregisterFrameId(this.rootFrameId); + }); + + this.page.on("framenavigated", async (frame) => { + if (frame.parentFrame() === null) { + const { frameTree: ft } = + await this.sendCDP( + "Page.getFrameTree", + ); + const newId = ft.frame.id; + if (newId !== this.rootFrameId) { + this.intContext.unregisterFrameId(this.rootFrameId); + this.rootFrameId = newId; + this.intContext.registerFrameId(this.rootFrameId, this); + } + } + }); return this; } catch (err: unknown) { if (err instanceof StagehandError || err instanceof StagehandAPIError) { From 2668f77718436dfe8251b30d60e6425ba1d345c2 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Sat, 19 Jul 2025 15:42:03 -0700 Subject: [PATCH 02/12] include frameID in api calls --- lib/StagehandPage.ts | 9 ++++++--- types/playwright.ts | 1 + types/stagehand.ts | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts index 67bedb6df..29f3c4813 100644 --- a/lib/StagehandPage.ts +++ b/lib/StagehandPage.ts @@ -692,7 +692,8 @@ ${scriptContent} \ const { action, modelName, modelClientOptions } = actionOrOptions; if (this.api) { - const result = await this.api.act(actionOrOptions); + const opts = { ...actionOrOptions, frameId: this.frameId }; + const result = await this.api.act(opts); await this._refreshPageFromAPI(); this.stagehand.addToHistory("act", actionOrOptions, result); return result; @@ -790,7 +791,8 @@ ${scriptContent} \ } if (this.api) { - const result = await this.api.extract(options); + const opts = { ...options, frameId: this.frameId }; + const result = await this.api.extract(opts); this.stagehand.addToHistory("extract", instructionOrOptions, result); return result; } @@ -897,7 +899,8 @@ ${scriptContent} \ } if (this.api) { - const result = await this.api.observe(options); + const opts = { ...options, frameId: this.frameId }; + const result = await this.api.observe(opts); this.stagehand.addToHistory("observe", instructionOrOptions, result); return result; } diff --git a/types/playwright.ts b/types/playwright.ts index bb623576b..8ec437cf7 100644 --- a/types/playwright.ts +++ b/types/playwright.ts @@ -16,4 +16,5 @@ export interface GotoOptions { timeout?: number; waitUntil?: "load" | "domcontentloaded" | "networkidle" | "commit"; referer?: string; + frameId?: string; } diff --git a/types/stagehand.ts b/types/stagehand.ts index d6ed42d85..706a45a3b 100644 --- a/types/stagehand.ts +++ b/types/stagehand.ts @@ -116,6 +116,7 @@ export interface ActOptions { domSettleTimeoutMs?: number; timeoutMs?: number; iframes?: boolean; + frameId?: string; } export interface ActResult { @@ -136,6 +137,7 @@ export interface ExtractOptions { useTextExtract?: boolean; selector?: string; iframes?: boolean; + frameId?: string; } export type ExtractResult = z.infer; @@ -152,6 +154,7 @@ export interface ObserveOptions { onlyVisible?: boolean; drawOverlay?: boolean; iframes?: boolean; + frameId?: string; } export interface ObserveResult { From 707eb292dd1953e059e1e7ca663fe27510201231 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 22 Jul 2025 10:29:46 -0700 Subject: [PATCH 03/12] add frame ID to goto, extract, and act from observe result --- lib/StagehandPage.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts index 29f3c4813..f87bbecb7 100644 --- a/lib/StagehandPage.ts +++ b/lib/StagehandPage.ts @@ -362,7 +362,10 @@ ${scriptContent} \ return async (url: string, options: GotoOptions) => { this.intContext.setActivePage(this); const result = this.api - ? await this.api.goto(url, options) + ? await this.api.goto(url, { + ...options, + frameId: this.frameId, + }) : await rawGoto(url, options); this.stagehand.addToHistory("navigate", { url, options }, result); @@ -660,7 +663,10 @@ ${scriptContent} \ const observeResult = actionOrOptions as ObserveResult; if (this.api) { - const result = await this.api.act(observeResult); + const result = await this.api.act({ + ...observeResult, + frameId: this.frameId, + }); await this._refreshPageFromAPI(); this.stagehand.addToHistory("act", observeResult, result); return result; @@ -754,7 +760,7 @@ ${scriptContent} \ if (!instructionOrOptions) { let result: ExtractResult; if (this.api) { - result = await this.api.extract({}); + result = await this.api.extract({frameId: this.frameId}); } else { result = await this.extractHandler.extract(); } From 44969a17e920c2346eab8174591fdc9ad36ef4b8 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 22 Jul 2025 11:19:37 -0700 Subject: [PATCH 04/12] prettier --- lib/StagehandPage.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts index f87bbecb7..5eb0414aa 100644 --- a/lib/StagehandPage.ts +++ b/lib/StagehandPage.ts @@ -363,9 +363,9 @@ ${scriptContent} \ this.intContext.setActivePage(this); const result = this.api ? await this.api.goto(url, { - ...options, - frameId: this.frameId, - }) + ...options, + frameId: this.frameId, + }) : await rawGoto(url, options); this.stagehand.addToHistory("navigate", { url, options }, result); @@ -760,7 +760,7 @@ ${scriptContent} \ if (!instructionOrOptions) { let result: ExtractResult; if (this.api) { - result = await this.api.extract({frameId: this.frameId}); + result = await this.api.extract({ frameId: this.frameId }); } else { result = await this.extractHandler.extract(); } From 355b1f0c10a7c965c3e18c4a2ac4f5c6b65c4809 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 22 Jul 2025 12:19:48 -0700 Subject: [PATCH 05/12] dont set active page on goto --- lib/StagehandPage.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts index 5eb0414aa..e030703a0 100644 --- a/lib/StagehandPage.ts +++ b/lib/StagehandPage.ts @@ -360,7 +360,6 @@ ${scriptContent} \ const rawGoto: typeof target.goto = Object.getPrototypeOf(target).goto.bind(target); return async (url: string, options: GotoOptions) => { - this.intContext.setActivePage(this); const result = this.api ? await this.api.goto(url, { ...options, From 58b0848269a3baca2fac145b0299cf1c39d7ff40 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 22 Jul 2025 13:49:43 -0700 Subject: [PATCH 06/12] add options to extract with no args --- lib/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api.ts b/lib/api.ts index 6689f4633..de60d2b8d 100644 --- a/lib/api.ts +++ b/lib/api.ts @@ -123,7 +123,7 @@ export class StagehandAPI { if (!options.schema) { return this.execute>({ method: "extract", - args: {}, + args: { ...options }, }); } const parsedSchema = zodToJsonSchema(options.schema); From 2924d713905112ebbffbdc49572080fb921b050f Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 22 Jul 2025 17:32:49 -0700 Subject: [PATCH 07/12] changeset --- .changeset/swift-roses-design.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/swift-roses-design.md diff --git a/.changeset/swift-roses-design.md b/.changeset/swift-roses-design.md new file mode 100644 index 000000000..ed3aaa685 --- /dev/null +++ b/.changeset/swift-roses-design.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +store mapping of CDP frame ID -> page From 2a437b98fedb1df3bbc82f106ef99906adf8052d Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Wed, 23 Jul 2025 12:08:38 -0700 Subject: [PATCH 08/12] use CDP listener instead of playwright --- lib/StagehandContext.ts | 57 ++++++++++++++++++++++++++++++++++++----- lib/StagehandPage.ts | 29 +++------------------ 2 files changed, 54 insertions(+), 32 deletions(-) diff --git a/lib/StagehandContext.ts b/lib/StagehandContext.ts index 92062c3e9..f93186f1e 100644 --- a/lib/StagehandContext.ts +++ b/lib/StagehandContext.ts @@ -1,11 +1,13 @@ import type { BrowserContext as PlaywrightContext, + CDPSession, Page as PlaywrightPage, } from "playwright"; import { Stagehand } from "./index"; import { StagehandPage } from "./StagehandPage"; import { Page } from "../types/page"; import { EnhancedContext } from "../types/context"; +import { Protocol } from "devtools-protocol"; export class StagehandContext { private readonly stagehand: Stagehand; @@ -84,6 +86,7 @@ export class StagehandContext { const existingPages = context.pages(); for (const page of existingPages) { const stagehandPage = await instance.createStagehandPage(page); + await instance.attachFrameNavigatedListener(page); // Set the first page as active if (!instance.activeStagehandPage) { instance.setActivePage(stagehandPage); @@ -91,13 +94,24 @@ export class StagehandContext { } context.on("page", (pwPage) => { - instance.handleNewPlaywrightPage(pwPage).catch((err) => - stagehand.logger({ - category: "context", - message: `Failed to initialise new page: ${err}`, - level: 0, - }), - ); + instance + .attachFrameNavigatedListener(pwPage) + .catch((err) => + stagehand.logger({ + category: "cdp", + message: `Failed to attach frameNavigated listener: ${err}`, + level: 0, + }), + ) + .finally(() => + instance.handleNewPlaywrightPage(pwPage).catch((err) => + stagehand.logger({ + category: "context", + message: `Failed to initialise new page: ${err}`, + level: 0, + }), + ), + ); }); return instance; @@ -156,4 +170,33 @@ export class StagehandContext { } this.setActivePage(stagehandPage); } + + private async attachFrameNavigatedListener( + pwPage: PlaywrightPage, + ): Promise { + const shPage = this.pageMap.get(pwPage); + if (!shPage) return; + const session: CDPSession = await this.intContext.newCDPSession(pwPage); + await session.send("Page.enable"); + + pwPage.once("close", () => { + this.unregisterFrameId(shPage.frameId); + }); + + session.on( + "Page.frameNavigated", + (evt: Protocol.Page.FrameNavigatedEvent): void => { + const { frame } = evt; + + if (!frame.parentId) { + const oldId = shPage.frameId; + if (frame.id !== oldId) { + if (oldId) this.unregisterFrameId(oldId); + this.registerFrameId(frame.id, shPage); + shPage.updateRootFrameId(frame.id); + } + } + }, + ); + } } diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts index e030703a0..d4cce467b 100644 --- a/lib/StagehandPage.ts +++ b/lib/StagehandPage.ts @@ -62,6 +62,10 @@ export class StagehandPage { return this.rootFrameId; } + public updateRootFrameId(newId: string): void { + this.rootFrameId = newId; + } + constructor( page: PlaywrightPage, stagehand: Stagehand, @@ -434,31 +438,6 @@ ${scriptContent} \ this.intPage = new Proxy(page, handler) as unknown as Page; this.initialized = true; - const { frameTree } = - await this.sendCDP( - "Page.getFrameTree", - ); - this.rootFrameId = frameTree.frame.id; - this.intContext.registerFrameId(this.rootFrameId, this); - - this.page.once("close", () => { - this.intContext.unregisterFrameId(this.rootFrameId); - }); - - this.page.on("framenavigated", async (frame) => { - if (frame.parentFrame() === null) { - const { frameTree: ft } = - await this.sendCDP( - "Page.getFrameTree", - ); - const newId = ft.frame.id; - if (newId !== this.rootFrameId) { - this.intContext.unregisterFrameId(this.rootFrameId); - this.rootFrameId = newId; - this.intContext.registerFrameId(this.rootFrameId, this); - } - } - }); return this; } catch (err: unknown) { if (err instanceof StagehandError || err instanceof StagehandAPIError) { From 7898275a4b255777057e1fe84af7c09b20f9b108 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Wed, 23 Jul 2025 17:24:11 -0700 Subject: [PATCH 09/12] remove unused frameId params --- types/stagehand.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/types/stagehand.ts b/types/stagehand.ts index 706a45a3b..d6ed42d85 100644 --- a/types/stagehand.ts +++ b/types/stagehand.ts @@ -116,7 +116,6 @@ export interface ActOptions { domSettleTimeoutMs?: number; timeoutMs?: number; iframes?: boolean; - frameId?: string; } export interface ActResult { @@ -137,7 +136,6 @@ export interface ExtractOptions { useTextExtract?: boolean; selector?: string; iframes?: boolean; - frameId?: string; } export type ExtractResult = z.infer; @@ -154,7 +152,6 @@ export interface ObserveOptions { onlyVisible?: boolean; drawOverlay?: boolean; iframes?: boolean; - frameId?: string; } export interface ObserveResult { From 582998a6768c24315dce35eb06cc92ba21d8948d Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Wed, 23 Jul 2025 17:54:45 -0700 Subject: [PATCH 10/12] revert removal of params --- types/stagehand.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/types/stagehand.ts b/types/stagehand.ts index d6ed42d85..706a45a3b 100644 --- a/types/stagehand.ts +++ b/types/stagehand.ts @@ -116,6 +116,7 @@ export interface ActOptions { domSettleTimeoutMs?: number; timeoutMs?: number; iframes?: boolean; + frameId?: string; } export interface ActResult { @@ -136,6 +137,7 @@ export interface ExtractOptions { useTextExtract?: boolean; selector?: string; iframes?: boolean; + frameId?: string; } export type ExtractResult = z.infer; @@ -152,6 +154,7 @@ export interface ObserveOptions { onlyVisible?: boolean; drawOverlay?: boolean; iframes?: boolean; + frameId?: string; } export interface ObserveResult { From c55ae456b157a4e782a12634df8244e3173d380a Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Wed, 23 Jul 2025 18:55:25 -0700 Subject: [PATCH 11/12] update test --- .../tests/BrowserContext/multiPage.test.ts | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/evals/deterministic/tests/BrowserContext/multiPage.test.ts b/evals/deterministic/tests/BrowserContext/multiPage.test.ts index 041cf1c3f..03264be4b 100644 --- a/evals/deterministic/tests/BrowserContext/multiPage.test.ts +++ b/evals/deterministic/tests/BrowserContext/multiPage.test.ts @@ -128,26 +128,24 @@ test.describe("StagehandContext - Multi-page Support", () => { * Test popup handling */ test("should handle popups with enhanced capabilities", async () => { - const mainPage = stagehand.page; - let popupPage: Page | null = null; + await stagehand.page.goto(`http://localhost:${serverPort}/page1`); + await stagehand.page.click("#popupBtn"); - mainPage.on("popup", (page: Page) => { - popupPage = page; - }); + await expect.poll(() => stagehand.context.pages().length).toBe(2); - await mainPage.goto(`http://localhost:${serverPort}/page1`); - await mainPage.click("#popupBtn"); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const [_original, popupPage] = stagehand.context.pages(); - // Verify popup has enhanced capabilities - expect(popupPage).not.toBeNull(); - expect(typeof popupPage.act).toBe("function"); - expect(typeof popupPage.extract).toBe("function"); - expect(typeof popupPage.observe).toBe("function"); + await popupPage.waitForLoadState(); - if (popupPage) { - await popupPage.waitForLoadState(); - expect(await popupPage.title()).toBe("Page 2"); - } + const get = (k: string) => + (popupPage as unknown as Record)[k]; + + expect(typeof get("act")).toBe("function"); + expect(typeof get("extract")).toBe("function"); + expect(typeof get("observe")).toBe("function"); + + expect(await popupPage.title()).toBe("Page 2"); }); /** From 78940d4bee974002b3d079dad81edb8bfda6c754 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Wed, 23 Jul 2025 19:02:19 -0700 Subject: [PATCH 12/12] rm unused import --- evals/deterministic/tests/BrowserContext/multiPage.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/evals/deterministic/tests/BrowserContext/multiPage.test.ts b/evals/deterministic/tests/BrowserContext/multiPage.test.ts index 03264be4b..87c5cfc8c 100644 --- a/evals/deterministic/tests/BrowserContext/multiPage.test.ts +++ b/evals/deterministic/tests/BrowserContext/multiPage.test.ts @@ -1,7 +1,6 @@ import { test, expect } from "@playwright/test"; import { Stagehand } from "@browserbasehq/stagehand"; import StagehandConfig from "@/evals/deterministic/stagehand.config"; -import { Page } from "@browserbasehq/stagehand"; import http from "http"; import express from "express";