Browser agent

from playwright.sync_api import sync_playwright
from simoai import Client, Choice, Extract, Fill, Locate, Noul, image

c = Client()
TASK = "Subscribe to the newsletter with the email ada@example.com"

QUESTIONS = {
    "done": Noul("The task is complete"),
    "next": Choice("What should the agent do next to complete the task?", {
        "click":  "click an element on the page",
        "type":   "type text into the focused field",
        "scroll": "scroll down to see more of the page",
        "stop":   "the task is complete or cannot be done",
    }, args={
        "click": {"target": Extract("the visible label of the element to click", max_tokens=16)},
        "type":  {"text": Fill("the exact text to type", max_tokens=32)},
    }),
    "where": Locate("the element the agent should click or type into next"),
}

with sync_playwright() as pw:
    page = pw.chromium.launch().new_page(viewport={"width": 1280, "height": 800})
    page.goto("https://example.com/newsletter")
    history = []

    for step in range(15):
        shot = page.screenshot()
        r = c.ask({"task": TASK, "history": "\n".join(history) or "(nothing yet)", "screen": "<image:1>"},
                  QUESTIONS, media=[image(shot)])

        if r["done"].noul > 0.9:
            print("done after", step, "steps")
            break
        nxt = r["next"]
        if nxt.probabilities[nxt.choice] < 0.75:
            print("not sure what to do next:", nxt.probabilities)
            break

        if nxt.choice == "click" and r["where"].center:
            x, y = r["where"].center
            page.mouse.click(x, y)
        elif nxt.choice == "type":
            if r["where"].center:
                page.mouse.click(*r["where"].center)
            page.keyboard.type(nxt.arg_values["text"])
        elif nxt.choice == "scroll":
            page.mouse.wheel(0, 600)
        else:
            break
        history.append(f"{nxt.choice} {nxt.arg_values}")

import { chromium } from "playwright";

const TASK = "Subscribe to the newsletter with the email ada@example.com";
const QUESTIONS = {
  done: { type: "noul", instructions: "The task is complete" },
  next: {
    type: "choice",
    instructions: "What should the agent do next to complete the task?",
    criteria: {
      click: "click an element on the page",
      type: "type text into the focused field",
      scroll: "scroll down to see more of the page",
      stop: "the task is complete or cannot be done",
    },
    args: {
      click: { target: { type: "extract", instructions: "the visible label of the element to click", kind: "text", many: false, max_tokens: 16 } },
      type: { text: { type: "fill", instructions: "the exact text to type", kind: "text", max_tokens: 32 } },
    },
  },
  where: { type: "extract", kind: "coordinates", instructions: "the element the agent should click or type into next", image: 1, scale: "rel1000", max_tokens: 64 },
};

const browser = await chromium.launch();
const page = await browser.newPage({ viewport: { width: 1280, height: 800 } });
await page.goto("https://example.com/newsletter");
const history = [];

for (let step = 0; step < 15; step++) {
  const shot = (await page.screenshot()).toString("base64");
  const res = await fetch("https://api.temprl.org/v1/ask", {
    method: "POST",
    headers: { Authorization: `Bearer ${process.env.SIMO_API_KEY}`, "Content-Type": "application/json" },
    body: JSON.stringify({
      model: "simo-1-pro",
      state: { task: TASK, history: history.join("\n") || "(nothing yet)", screen: "<image:1>" },
      media: [{ type: "image", data: `data:image/png;base64,${shot}` }],
      questions: QUESTIONS,
    }),
  });
  const { answers: a } = await res.json();
  if (a.done.noul > 0.9) break;
  if (a.next.probabilities[a.next.choice] < 0.75) break;   // unsure: hand over

  if (a.next.choice === "click" && a.where.center) await page.mouse.click(...a.where.center);
  else if (a.next.choice === "type") {
    if (a.where.center) await page.mouse.click(...a.where.center);
    await page.keyboard.type(a.next.args.text.value);
  } else if (a.next.choice === "scroll") await page.mouse.wheel(0, 600);
  else break;
  history.push(`${a.next.choice} ${JSON.stringify(a.next.args ?? {})}`);
}
await browser.close();