@tool(atomic=True)
def fetch_page(
    url: str,
    extract: str = "text",
    max_length: int = 5000,
) -> str:
    """Fetch a web page and extract its content.

    Retrieves the page at the given URL and returns readable text content.
    Use this to read articles, documentation, reference pages, or any web content.
    Does NOT execute JavaScript — works best with static content, articles, docs.

    Args:
        url: The full URL to fetch (must start with http:// or https://)
        extract: What to extract - 'text' (readable content), 'html' (raw HTML),
                 'links' (all links on page), 'tables' (HTML tables as text)
        max_length: Maximum characters to return (default: 5000, max: 20000)
    """

Page: Example Documentation - My Library
URL: https://example.com/docs/api
Length: 4,521 chars | Fetched: 2026-03-10 14:30

API Reference
=============

Authentication
--------------
All API requests require a Bearer token in the Authorization header.

Endpoints
---------
GET /api/users - List all users
POST /api/users - Create a new user
...

@tool(atomic=True)
def search_web(
    query: str,
    num_results: int = 5,
) -> str:
    """Search the web and return results with titles, URLs, and snippets.

    Uses a search API to find relevant web pages. Returns titles, URLs, and
    brief descriptions. Use fetch_page to read the full content of any result.

    Args:
        query: Search query string
        num_results: Number of results to return (default: 5, max: 10)
    """

Web search results for: "python sqlite fts5 tutorial"

1. SQLite FTS5 Full-Text Search - SQLite Documentation
   https://www.sqlite.org/fts5.html
   FTS5 is an SQLite virtual table module that provides full-text search...

2. Full-Text Search with SQLite and Python
   https://example.com/blog/sqlite-fts5-python
   Learn how to implement full-text search in Python using SQLite's FTS5...

3. ...

@tool(atomic=True)
def download_file(
    url: str,
    save_to: str = "~/Downloads",
    filename: str = None,
) -> str:
    """Download a file from a URL to the local filesystem.

    Downloads the file and saves it locally. Useful for getting documents,
    PDFs, CSVs, images, or any file from the web for local analysis.
    After downloading, use read_file or index_document to process it.

    Args:
        url: Direct URL to the file to download
        save_to: Local directory to save the file (default: ~/Downloads)
        filename: Override filename (default: derived from URL or Content-Disposition)
    """

Downloaded: report-2026.pdf
  Saved to: C:\Users\John\Downloads\report-2026.pdf
  Size: 2.4 MB
  Type: application/pdf

Use read_file or index_document to process this file.

ChatAgent
  |
  +-- BrowserToolsMixin (NEW - 3 tools)
  |     +-- fetch_page()           # Read web content (text/links/tables)
  |     +-- search_web()           # Web search
  |     +-- download_file()        # Download files to local disk
  |     |
  |     +-- self._web_client → WebClient (separate module)
  |           +-- get()            # HTTP GET with rate limiting + SSRF check
  |           +-- post()           # HTTP POST (for search)
  |           +-- parse_html()     # BeautifulSoup wrapper
  |           +-- extract_text()   # HTML to readable text
  |           +-- extract_tables() # HTML tables to JSON dicts
  |           +-- extract_links()  # Links extraction
  |           +-- download()       # Stream file to disk
  |
  +-- FileSystemToolsMixin (existing - 6 tools)
  +-- ScratchpadToolsMixin (existing - 5 tools)
  +-- RAGToolsMixin (existing)
  +-- ShellToolsMixin (existing)

class WebClient:
    """Lightweight HTTP client for web content extraction.

    Uses requests for HTTP and BeautifulSoup for HTML parsing.
    Handles rate limiting, timeouts, size limits, and content extraction.
    """

    DEFAULT_TIMEOUT = 30  # seconds
    DEFAULT_MAX_SIZE = 10 * 1024 * 1024  # 10 MB response limit
    MIN_REQUEST_INTERVAL = 1.0  # seconds between requests (rate limit)
    DEFAULT_USER_AGENT = "GAIA-Agent/0.15 (https://github.com/amd/gaia)"

    def __init__(self, timeout=None, max_size=None, user_agent=None):
        self._timeout = timeout or self.DEFAULT_TIMEOUT
        self._max_size = max_size or self.DEFAULT_MAX_SIZE
        self._user_agent = user_agent or self.DEFAULT_USER_AGENT
        self._last_request_time = 0  # For rate limiting
        self._session = requests.Session()
        self._session.headers.update({
            "User-Agent": self._user_agent,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.5",
        })

    def get(self, url: str, stream: bool = False) -> requests.Response:
        """HTTP GET with rate limiting, timeout, and size checking."""

    def parse_html(self, html: str) -> BeautifulSoup:
        """Parse HTML content."""

    def extract_text(self, soup: BeautifulSoup, max_length: int = 5000) -> str:
        """Extract readable text from parsed HTML."""

    def extract_tables(self, soup: BeautifulSoup) -> list[list[dict]]:
        """Extract HTML tables as list of list-of-dicts."""

    def extract_links(self, soup: BeautifulSoup, base_url: str) -> list[dict]:
        """Extract all links with text and resolved URLs."""

    def close(self):
        """Close the session."""

src/gaia/web/
+-- __init__.py               # Exports WebClient
+-- client.py                 # WebClient (HTTP + HTML extraction)

src/gaia/agents/tools/
+-- browser_tools.py          # BrowserToolsMixin (3 tools, delegates to WebClient)

class ChatAgent(
    Agent,
    RAGToolsMixin,
    FileToolsMixin,
    ShellToolsMixin,
    FileSystemToolsMixin,
    ScratchpadToolsMixin,
    BrowserToolsMixin,         # NEW
):

@dataclass
class ChatAgentConfig:
    # ... existing fields ...

    # Browser settings
    enable_browser: bool = True  # Enable web browsing tools
    browser_timeout: int = 30  # HTTP request timeout in seconds
    browser_max_download_size: int = 100 * 1024 * 1024  # 100 MB max download
    browser_user_agent: str = "GAIA-Agent/0.15"
    browser_rate_limit: float = 1.0  # Seconds between requests

def _register_tools(self) -> None:
    self.register_rag_tools()
    self.register_file_tools()
    self.register_shell_tools()
    self.register_filesystem_tools()
    self.register_scratchpad_tools()
    self.register_browser_tools()  # NEW

User: "Compare my monthly grocery spending to the national average"

Agent:
1. query_data("SELECT strftime('%Y-%m', date) as month, SUM(amount) as total
               FROM scratch_transactions WHERE category='groceries' GROUP BY month")
   → User spends ~$650/month on groceries

2. search_web("average monthly grocery spending US household 2026")
   → Finds USDA data page

3. fetch_page("https://www.usda.gov/food-spending-data")
   → Extracts: "Average US household: $475/month"

4. Answer: "Your average monthly grocery spending is $650, which is 37% above
   the national average of $475/month. Here's the month-by-month breakdown..."

User: "Download the latest AMD earnings report and summarize it"

Agent:
1. search_web("AMD Q4 2025 earnings report PDF")
   → Finds direct PDF link

2. download_file("https://ir.amd.com/reports/Q4-2025.pdf")
   → Saved to ~/Downloads/Q4-2025.pdf

3. index_document("~/Downloads/Q4-2025.pdf")
   → Indexed, 85 chunks

4. query_documents("key financial metrics revenue profit")
   → Extracts: Revenue $7.1B, Net Income $1.2B...

5. Answer: "AMD's Q4 2025 earnings report shows..."

User: "Scrape the pricing from these three SaaS competitors and compare"

Agent:
1. extract_page_data("https://competitor1.com/pricing")
   → JSON table of plans

2. create_table("competitor_pricing",
     "company TEXT, plan TEXT, price_monthly REAL, users INTEGER, features TEXT")

3. insert_data("competitor_pricing", [...extracted data...])

4. Repeat for competitors 2 and 3

5. query_data("SELECT company, plan, price_monthly FROM scratch_competitor_pricing
               ORDER BY price_monthly")

6. Answer: "Here's a comparison of all three competitors' pricing..."

import ipaddress
import socket
from urllib.parse import urlparse

ALLOWED_SCHEMES = {"http", "https"}
BLOCKED_PORTS = {22, 23, 25, 445, 3306, 5432, 6379, 27017}  # SSH, SMTP, DB ports

def _validate_url(url: str) -> str:
    """Validate URL is safe to fetch. Returns normalized URL or raises ValueError.

    1. Parse URL and validate scheme (http/https only)
    2. Check port is not in blocked set
    3. Resolve hostname to IP address
    4. Validate resolved IP is not private/reserved/loopback/link-local
    5. Return validated URL
    """
    parsed = urlparse(url)
    if parsed.scheme not in ALLOWED_SCHEMES:
        raise ValueError(f"Blocked scheme: {parsed.scheme}")
    if parsed.port and parsed.port in BLOCKED_PORTS:
        raise ValueError(f"Blocked port: {parsed.port}")
    # Resolve and validate IP
    _validate_host_ip(parsed.hostname)
    return url

def _validate_host_ip(hostname: str) -> None:
    """Resolve hostname and check IP is not private/internal."""
    try:
        resolved = socket.getaddrinfo(hostname, None)
        for family, _, _, _, sockaddr in resolved:
            ip = ipaddress.ip_address(sockaddr[0])
            if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast:
                raise ValueError(f"Blocked: {hostname} resolves to private/reserved IP {ip}")
    except socket.gaierror:
        raise ValueError(f"Cannot resolve hostname: {hostname}")

def _sanitize_filename(raw_name: str) -> str:
    """Sanitize filename from URL or Content-Disposition header.

    1. Extract basename only (strip path components)
    2. Remove null bytes and control characters
    3. Replace path separators (/, \\) with _
    4. Reject filenames starting with . (hidden files)
    5. Limit to safe charset [a-zA-Z0-9._-]
    6. Truncate to 200 chars
    7. Fallback to 'download' if empty after sanitization
    """
    import re
    name = os.path.basename(raw_name)
    name = name.replace("\x00", "").strip()
    name = re.sub(r'[/\\]', '_', name)
    name = re.sub(r'[^a-zA-Z0-9._-]', '_', name)
    if name.startswith('.'):
        name = '_' + name
    name = name[:200]
    return name or "download"

def _search_duckduckgo(self, query: str, num_results: int = 5) -> list[dict]:
    """Search DuckDuckGo and parse results from HTML.

    Uses the HTML-only version (html.duckduckgo.com) which doesn't
    require JavaScript rendering.

    Returns list of {"title": str, "url": str, "snippet": str}.
    """
    response = self.get(
        "https://html.duckduckgo.com/html/",
        params={"q": query},
    )
    soup = self.parse_html(response.text)
    results = []
    for result in soup.select(".result"):
        title_el = result.select_one(".result__title a")
        snippet_el = result.select_one(".result__snippet")
        if title_el:
            results.append({
                "title": title_el.get_text(strip=True),
                "url": title_el.get("href", ""),
                "snippet": snippet_el.get_text(strip=True) if snippet_el else "",
            })
        if len(results) >= num_results:
            break
    return results

def extract_text(self, soup: BeautifulSoup, max_length: int = 5000) -> str:
    """Extract readable text, preserving structure.

    Strategy:
    1. Remove script, style, nav, footer, aside tags
    2. Preserve heading hierarchy (h1-h6 → underlined text)
    3. Preserve list structure (ul/ol → bulleted/numbered)
    4. Preserve paragraph breaks
    5. Collapse whitespace
    6. Truncate to max_length with word boundary
    """

REMOVE_TAGS = [
    "script", "style", "nav", "footer", "aside", "header",
    "noscript", "iframe", "svg", "form", "button", "input",
    "select", "textarea", "meta", "link",
]

def extract_tables(self, soup: BeautifulSoup) -> list:
    """Extract tables as list of dicts.

    For each <table>:
    1. Use first <tr> or <thead> as column headers
    2. Subsequent rows become dicts with header keys
    3. Strip whitespace from cells
    4. Skip tables with fewer than 2 rows (likely layout tables)
    """