# Browser agent

Source: https://docs.temprl.org/examples/browser-agent

> Drive a browser with Simo - read a screenshot, decide the next action with a calibrated probability, and locate the element to click in pixels.

Each step, take a screenshot and ask Simo three things at once: is the task done, what should happen next, and where is the thing to click. Simo answers with pixel coordinates of *your* screenshot, ready to pass to the browser. Below the confidence threshold, the agent stops and asks a person.

This example uses [Playwright](https://playwright.dev/python/); any tool that can screenshot and click works.

**Python**

```python
from playwright.sync_api import sync_playwright
from simoai import Client, Choice, Extract, Fill, Locate, Noul, image

c = Client()
TASK = "Subscribe to the newsletter with the email ada@example.com"

QUESTIONS = {
    "done": Noul("The task is complete"),
    "next": Choice("What should the agent do next to complete the task?", {
        "click":  "click an element on the page",
        "type":   "type text into the focused field",
        "scroll": "scroll down to see more of the page",
        "stop":   "the task is complete or cannot be done",
    }, args={
        "click": {"target": Extract("the visible label of the element to click", max_tokens=16)},
        "type":  {"text": Fill("the exact text to type", max_tokens=32)},
    }),
    "where": Locate("the element the agent should click or type into next"),
}

with sync_playwright() as pw:
    page = pw.chromium.launch().new_page(viewport={"width": 1280, "height": 800})
    page.goto("https://example.com/newsletter")
    history = []

    for step in range(15):
        shot = page.screenshot()
        r = c.ask({"task": TASK, "history": "\n".join(history) or "(nothing yet)", "screen": "<image:1>"},
                  QUESTIONS, media=[image(shot)])

        if r["done"].noul > 0.9:
            print("done after", step, "steps")
            break
        nxt = r["next"]
        if nxt.probabilities[nxt.choice] < 0.75:
            print("not sure what to do next:", nxt.probabilities)
            break

        if nxt.choice == "click" and r["where"].center:
            x, y = r["where"].center
            page.mouse.click(x, y)
        elif nxt.choice == "type":
            if r["where"].center:
                page.mouse.click(*r["where"].center)
            page.keyboard.type(nxt.arg_values["text"])
        elif nxt.choice == "scroll":
            page.mouse.wheel(0, 600)
        else:
            break
        history.append(f"{nxt.choice} {nxt.arg_values}")
```

**JavaScript**

```javascript
import { chromium } from "playwright";

const TASK = "Subscribe to the newsletter with the email ada@example.com";
const QUESTIONS = {
  done: { type: "noul", instructions: "The task is complete" },
  next: {
    type: "choice",
    instructions: "What should the agent do next to complete the task?",
    criteria: {
      click: "click an element on the page",
      type: "type text into the focused field",
      scroll: "scroll down to see more of the page",
      stop: "the task is complete or cannot be done",
    },
    args: {
      click: { target: { type: "extract", instructions: "the visible label of the element to click", kind: "text", many: false, max_tokens: 16 } },
      type: { text: { type: "fill", instructions: "the exact text to type", kind: "text", max_tokens: 32 } },
    },
  },
  where: { type: "extract", kind: "coordinates", instructions: "the element the agent should click or type into next", image: 1, scale: "rel1000", max_tokens: 64 },
};

const browser = await chromium.launch();
const page = await browser.newPage({ viewport: { width: 1280, height: 800 } });
await page.goto("https://example.com/newsletter");
const history = [];

for (let step = 0; step < 15; step++) {
  const shot = (await page.screenshot()).toString("base64");
  const res = await fetch("https://api.temprl.org/v1/ask", {
    method: "POST",
    headers: { Authorization: `Bearer ${process.env.SIMO_API_KEY}`, "Content-Type": "application/json" },
    body: JSON.stringify({
      model: "simo-1-pro",
      state: { task: TASK, history: history.join("\n") || "(nothing yet)", screen: "<image:1>" },
      media: [{ type: "image", data: `data:image/png;base64,${shot}` }],
      questions: QUESTIONS,
    }),
  });
  const { answers: a } = await res.json();
  if (a.done.noul > 0.9) break;
  if (a.next.probabilities[a.next.choice] < 0.75) break;   // unsure: hand over

  if (a.next.choice === "click" && a.where.center) await page.mouse.click(...a.where.center);
  else if (a.next.choice === "type") {
    if (a.where.center) await page.mouse.click(...a.where.center);
    await page.keyboard.type(a.next.args.text.value);
  } else if (a.next.choice === "scroll") await page.mouse.wheel(0, 600);
  else break;
  history.push(`${a.next.choice} ${JSON.stringify(a.next.args ?? {})}`);
}
await browser.close();
```

> **Note** Over raw HTTP, `center` is in the pixels of the image you sent. Send the screenshot as captured (no resizing) and the coordinates map straight onto the page. The Python SDK pads images for the model and maps answers back for you.

## Why it works

- **One screenshot, three answers.** Done-check, next action and target location share one read.
- **Stops instead of guessing.** A calibrated probability below 0.75 means the page is ambiguous; a person resolves it rather than the agent clicking at random.
- **History in the state.** Listing the steps taken so far keeps the agent from repeating itself.