Gemini Computer Use

# Copy to env.sh and source it before running.
export GEMINI_API_KEY=""

# Optional: Use a Playwright browser channel (e.g., chrome, msedge).
# Leave empty to use Playwright's bundled Chromium.
export COMPUTER_USE_BROWSER_CHANNEL=""

# Optional: Point to a Chromium-based browser executable (e.g., Brave).
# Takes precedence over COMPUTER_USE_BROWSER_CHANNEL if set.
export COMPUTER_USE_BROWSER_EXECUTABLE=""


# Gemini Computer Use Notes

- Model: `gemini-2.5-computer-use-preview-10-2025` (required when using the Computer Use tool).
- The model emits `function_call` actions that must be executed client-side.
- After each action, return a `function_response` with the latest screenshot + URL.
- If a response includes `safety_decision: require_confirmation`, you must ask the user to confirm before executing the action.

Supported actions (browser environment):
- open_web_browser
- wait_5_seconds
- go_back
- go_forward
- search
- navigate
- click_at
- hover_at
- type_text_at
- key_combination
- scroll_document
- scroll_at
- drag_and_drop


#!/usr/bin/env python3
import argparse
import os
import sys
import time
from typing import Any, Dict, List, Tuple

from playwright.sync_api import sync_playwright
from google import genai
from google.genai import types
from google.genai.types import Content, Part

MODEL_NAME = "gemini-2.5-computer-use-preview-10-2025"
DEFAULT_START_URL = "https://www.google.com"
DEFAULT_SCREEN_WIDTH = 1440
DEFAULT_SCREEN_HEIGHT = 900

SUPPORTED_ACTIONS = {
    "open_web_browser",
    "wait_5_seconds",
    "go_back",
    "go_forward",
    "search",
    "navigate",
    "click_at",
    "hover_at",
    "type_text_at",
    "key_combination",
    "scroll_document",
    "scroll_at",
    "drag_and_drop",
}


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Run a Gemini Computer Use browser automation loop via Playwright.",
    )
    parser.add_argument("--prompt", required=True, help="User goal to send to the model")
    parser.add_argument(
        "--start-url",
        default=DEFAULT_START_URL,
        help=f"Initial page to load (default: {DEFAULT_START_URL})",
    )
    parser.add_argument("--turn-limit", type=int, default=6, help="Max turns")
    parser.add_argument(
        "--headless",
        action="store_true",
        help="Run browser in headless mode",
    )
    parser.add_argument(
        "--screen-width",
        type=int,
        default=DEFAULT_SCREEN_WIDTH,
        help="Viewport width in pixels",
    )
    parser.add_argument(
        "--screen-height",
        type=int,
        default=DEFAULT_SCREEN_HEIGHT,
        help="Viewport height in pixels",
    )
    parser.add_argument(
        "--exclude",
        action="append",
        default=[],
        help="Exclude predefined Computer Use actions (can repeat)",
    )
    return parser.parse_args()


def require_env() -> str:
    api_key = os.getenv("GEMINI_API_KEY")
    if not api_key:
        print("Missing GEMINI_API_KEY. Export it before running.", file=sys.stderr)
        sys.exit(1)
    return api_key


def denormalize(value: int, size: int) -> int:
    return int(value / 1000 * size)


def normalize_keys(keys: str) -> str:
    mapping = {
        "ctrl": "Control",
        "control": "Control",
        "cmd": "Meta",
        "command": "Meta",
        "meta": "Meta",
        "alt": "Alt",
        "shift": "Shift",
        "enter": "Enter",
        "return": "Enter",
        "tab": "Tab",
        "backspace": "Backspace",
        "delete": "Delete",
        "esc": "Escape",
        "escape": "Escape",
        "space": "Space",
    }
    parts = [p.strip() for p in keys.split("+")]
    normalized_parts = []
    for part in parts:
        lower = part.lower()
        if lower in mapping:
            normalized_parts.append(mapping[lower])
        elif len(part) == 1:
            normalized_parts.append(part.upper())
        else:
            normalized_parts.append(part.capitalize())
    return "