Observe perform candidates

evals/evals.config.json

@@ -215,6 +215,22 @@
     {
       "name": "extract_zillow",
       "categories": ["text_extract"]
+    },
+    {
+      "name": "observe_github",
+      "categories": ["observe"]
+    },
+    {
+      "name": "observe_vantechjournal",
+      "categories": ["observe"]
+    },
+    {
+      "name": "observe_amazon_add_to_cart",
+      "categories": ["observe"]
+    },
+    {
+      "name": "observe_simple_google_search",
+      "categories": ["observe"]
     }
   ]
 }

evals/tasks/observe_amazon_add_to_cart.ts

@@ -0,0 +1,75 @@
+import { EvalFunction } from "@/types/evals";
+import { initStagehand } from "@/evals/initStagehand";
+import { performPlaywrightMethod } from "@/lib/a11y/utils";
+
+export const observe_amazon_add_to_cart: EvalFunction = async ({
+  modelName,
+  logger,
+}) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+
+  await stagehand.page.goto(
+    "https://www.amazon.com/Laptop-MacBook-Surface-Water-Resistant-Accessories/dp/B0D5M4H5CD",
+  );
+
+  await stagehand.page.waitForTimeout(5000);
+
+  const observations1 = await stagehand.page.observe({
+    instruction: "Find and click the 'Add to Cart' button",
+    useAccessibilityTree: true,
+    returnAction: true,
+  });
+
+  console.log(observations1);
+
+  // Example of using performPlaywrightMethod if you have the xpath
+  if (observations1.length > 0) {
+    const action1 = observations1[0];
+    await performPlaywrightMethod(
+      stagehand.page,
+      stagehand.logger,
+      action1.method,
+      action1.arguments,
+      action1.selector.replace("xpath=", ""),
+    );
+  }
+
+  await stagehand.page.waitForTimeout(2000);
+
+  const observations2 = await stagehand.page.observe({
+    instruction: "Find and click the 'Proceed to checkout' button",
+    useAccessibilityTree: true,
+    returnAction: true,
+  });
+
+  // Example of using performPlaywrightMethod if you have the xpath
+  if (observations2.length > 0) {
+    const action2 = observations2[0];
+    await performPlaywrightMethod(
+      stagehand.page,
+      stagehand.logger,
+      action2.method,
+      action2.arguments,
+      action2.selector.replace("xpath=", ""),
+    );
+  }
+  await stagehand.page.waitForTimeout(2000);
+
+  const currentUrl = stagehand.page.url();
+  const expectedUrlPrefix = "https://www.amazon.com/ap/signin";
+
+  await stagehand.close();
+
+  return {
+    _success: currentUrl.startsWith(expectedUrlPrefix),
+    currentUrl,
+    debugUrl,
+    sessionUrl,
+    logs: logger.getLogs(),
+  };
+};

evals/tasks/observe_github.ts

@@ -0,0 +1,74 @@
+import { initStagehand } from "@/evals/initStagehand";
+import { EvalFunction } from "@/types/evals";
+
+export const observe_github: EvalFunction = async ({
+  modelName,
+  logger,
+  useAccessibilityTree,
+}) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+
+  await stagehand.page.goto(
+    "https://github.com/browserbase/stagehand/tree/main/lib",
+  );
+
+  const observations = await stagehand.page.observe({
+    instruction: "find the scrollable element that holds the repos file tree",
+    useAccessibilityTree,
+  });
+
+  if (observations.length === 0) {
+    await stagehand.close();
+    return {
+      _success: false,
+      observations,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  }
+
+  const expectedLocator = `#repos-file-tree > div.Box-sc-g0xbh4-0.jbQqON > div > div > div > nav > ul`;
+
+  const expectedResult = await stagehand.page
+    .locator(expectedLocator)
+    .first()
+    .innerText();
+
+  let foundMatch = false;
+  for (const observation of observations) {
+    try {
+      const observationResult = await stagehand.page
+        .locator(observation.selector)
+        .first()
+        .innerText();
+
+      if (observationResult === expectedResult) {
+        foundMatch = true;
+        break;
+      }
+    } catch (error) {
+      console.warn(
+        `Failed to check observation with selector ${observation.selector}:`,
+        error.message,
+      );
+      continue;
+    }
+  }
+
+  await stagehand.close();
+
+  return {
+    _success: foundMatch,
+    expected: expectedResult,
+    observations,
+    debugUrl,
+    sessionUrl,
+    logs: logger.getLogs(),
+  };
+};

evals/tasks/observe_simple_google_search.ts

@@ -0,0 +1,70 @@
+import { EvalFunction } from "@/types/evals";
+import { initStagehand } from "@/evals/initStagehand";
+import { performPlaywrightMethod } from "@/lib/a11y/utils";
+
+export const observe_simple_google_search: EvalFunction = async ({
+  modelName,
+  logger,
+}) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+
+  await stagehand.page.goto("https://www.google.com");
+
+  // await stagehand.page.act({
+  //   action: 'Search for "OpenAI"',
+  // });
+  const observation1 = await stagehand.page.observe({
+    instruction: "Find the search bar and enter 'OpenAI'",
+    useAccessibilityTree: true,
+    returnAction: true,
+  });
+  console.log(observation1);
+
+  if (observation1.length > 0) {
+    const action1 = observation1[0];
+    await performPlaywrightMethod(
+      stagehand.page,
+      stagehand.logger,
+      action1.method,
+      action1.arguments,
+      action1.selector.replace("xpath=", ""),
+    );
+  }
+  await stagehand.page.waitForTimeout(5000);
+  const observation2 = await stagehand.page.observe({
+    instruction: "Click the search button in the suggestions dropdown",
+    useAccessibilityTree: true,
+    returnAction: true,
+  });
+  console.log(observation2);
+
+  if (observation2.length > 0) {
+    const action2 = observation2[0];
+    await performPlaywrightMethod(
+      stagehand.page,
+      stagehand.logger,
+      action2.method,
+      action2.arguments,
+      action2.selector.replace("xpath=", ""),
+    );
+  }
+  await stagehand.page.waitForTimeout(5000);
+
+  const expectedUrl = "https://www.google.com/search?q=OpenAI";
+  const currentUrl = stagehand.page.url();
+
+  await stagehand.close();
+
+  return {
+    _success: currentUrl.startsWith(expectedUrl),
+    currentUrl,
+    debugUrl,
+    sessionUrl,
+    logs: logger.getLogs(),
+  };
+};

evals/tasks/observe_vantechjournal.ts

@@ -0,0 +1,72 @@
+import { initStagehand } from "@/evals/initStagehand";
+import { EvalFunction } from "@/types/evals";
+
+export const observe_vantechjournal: EvalFunction = async ({
+  modelName,
+  logger,
+  useAccessibilityTree,
+}) => {
+  const { stagehand, initResponse } = await initStagehand({
+    modelName,
+    logger,
+  });
+
+  const { debugUrl, sessionUrl } = initResponse;
+
+  await stagehand.page.goto("https://vantechjournal.com/archive?page=8");
+
+  const observations = await stagehand.page.observe({
+    instruction: "find the button that takes us to the 11th page",
+    useAccessibilityTree,
+  });
+
+  if (observations.length === 0) {
+    await stagehand.close();
+    return {
+      _success: false,
+      observations,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  }
+
+  const expectedLocator = `a.rounded-lg:nth-child(8)`;
+
+  const expectedResult = await stagehand.page
+    .locator(expectedLocator)
+    .first()
+    .innerText();
+
+  let foundMatch = false;
+  for (const observation of observations) {
+    try {
+      const observationResult = await stagehand.page
+        .locator(observation.selector)
+        .first()
+        .innerText();
+
+      if (observationResult === expectedResult) {
+        foundMatch = true;
+        break;
+      }
+    } catch (error) {
+      console.warn(
+        `Failed to check observation with selector ${observation.selector}:`,
+        error.message,
+      );
+      continue;
+    }
+  }
+
+  await stagehand.close();
+
+  return {
+    _success: foundMatch,
+    expected: expectedResult,
+    observations,
+    debugUrl,
+    sessionUrl,
+    logs: logger.getLogs(),
+  };
+};

lib/StagehandPage.ts

@@ -471,12 +471,13 @@ export class StagehandPage {
         : instructionOrOptions || {};
 
     const {
-      instruction = "Find actions that can be performed on this page.",
+      instruction,
       modelName,
       modelClientOptions,
       useVision, // still destructure but will not pass it on
       domSettleTimeoutMs,
-      useAccessibilityTree = false,
+      returnAction = true,
+      useAccessibilityTree = true,
     } = options;
 
     if (typeof useVision !== "undefined") {
@@ -523,6 +524,7 @@ export class StagehandPage {
         llmClient,
         requestId,
         domSettleTimeoutMs,
+        returnAction,
         useAccessibilityTree,
       })
       .catch((e) => {