"""web — search, fetch, run: the web, Python-shaped. Modes: - ``search`` — ONE query to SearXNG -> structured hits (url, title, content, score, engines) plus the answers, infoboxes, suggestions, corrections and DOWN ENGINES SearXNG also returns and the MCP tool discarded. - ``fetch`` — ONE URL -> the whole page: ``.markdown`` (readability + markdownify) and ``.html`` (raw), with ``.text`` a windowed rendering. - ``run`` — ONE URL in a real headless chromium -> the same PageResult, but ``.rendered`` and ``.html`` is the DOM after the page's JavaScript ran. Plus ``.screenshot`` (a VisionResult in the ImageStore, so a vision model SEES the page) and ``.page`` (the live playwright Page). - ``close`` — shut the kernel's browser down. One query per call and no pagination, for the same reason: an MCP tool has to return ONE STRING, so web_search took a list of queries and looped over them SEQUENTIALLY, and web_fetch sliced pages with start_index/max_length. In a kernel the model writes ``asyncio.gather(*[web("search", q) for q in qs])`` for real parallelism and ``r.markdown[5000:9000]`` for the next page. Bodies are STREAMED against a cap, so a 2GB download is refused at 10MB instead of after it has landed in RAM — the lesson ripgrep taught fs. A non-text content type raises rather than decoding a PDF into mojibake, and the error says what to do instead: httpx is ambient, so two lines of cell code download the bytes with nothing lost. Extraction prefers Mozilla's Readability.js, which readabilipy drives through ``node`` — measurably better than its pure-Python mode at ~0.8s instead of ~0.1s. Availability is decided HERE, with shutil.which and a directory stat, rather than by readabilipy's own ``have_node()``: that function runs ``npm install`` when its node_modules is missing, and npm install does a PROCESS-WIDE chdir — a network install and a cwd race, triggered lazily by a fetch, in a kernel where other cells resolve relative paths. Without node, extraction falls back to pure Python; crow strips the ``
tags. The title # is already reported separately; it does not belong in the body. soup = BeautifulSoup(article, "html.parser") for junk in soup.find_all("title"): junk.decompose() article = str(soup) if not article.strip(): return body.strip(), title, False markdown = markdownify.markdownify(article, heading_style=markdownify.ATX) return markdown.strip(), title, True async def _fetch(url: str, user_agent: str) -> PageResult: try: async with AsyncClient(follow_redirects=True, timeout=_TIMEOUT) as client: async with client.stream( "GET", url, headers={"User-Agent": user_agent} ) as response: status = response.status_code content_type = response.headers.get("content-type", "") final = str(response.url) if status >= 400: raise WebError( f"{status} {response.reason_phrase} for {final}" ) if not _is_texty(content_type): raise WebError( f"{final} is {_mime(content_type) or 'an undeclared type'}," " not text — web('fetch') reads pages. httpx is ambient:" " async with httpx.AsyncClient() as c:" " Path('f.pdf').write_bytes((await c.get(url)).content)" ) chunks: list[bytes] = [] size = 0 async for chunk in response.aiter_bytes(): size += len(chunk) if size > _MAX_PAGE_BYTES: raise WebError( f"{final} is over" f" {_MAX_PAGE_BYTES // (1024 * 1024)}MB — refusing to" " read the rest of it" ) chunks.append(chunk) encoding = response.encoding or "utf-8" except WebError: raise except HTTPError as e: raise WebError(f"fetching {url} failed: {e}") from None body = b"".join(chunks).decode(encoding, errors="replace") markdown, title, extracted = await asyncio.to_thread(_extract, body, content_type) return PageResult( url=final, title=title, status=status, content_type=content_type, markdown=markdown, html=body, extracted=extracted, ) async def _shutdown() -> bool: """Tear the browser down; report whether one was running. Best effort by necessity: a browser that crashed raises when you close it, and close() still has to get rid of the driver subprocess — so every layer is attempted, page then context then browser then driver, and a failure at one layer does not strand the next. """ was_running = _state.get("browser") is not None for key, method in ( ("page", "close"), ("context", "close"), ("browser", "close"), ("pw", "stop"), ): obj = _state.get(key) _state[key] = None if obj is None: continue with suppress(Exception): await getattr(obj, method)() return was_running async def _browser_page(): """A fresh page in the kernel's one context, closing the previous page. ONE browser and ONE context for the kernel's lifetime — a launch per cell costs ~0.3s and leaks processes, and the context is what makes cookies and localStorage survive from one run to the next. But a NEW page per run, with the previous one closed, because a shared page is a silent trap: verified live, ``r.page`` from an earlier cell was pointing at whatever the LATEST run had navigated to, so a selector that happened to exist on both pages clicked the wrong thing and one that did not burned a 30s timeout. Closing it makes a stale ``.page`` fail loudly with playwright's own "Page has been closed". """ from playwright.async_api import Error as PlaywrightError from playwright.async_api import async_playwright try: if _state.get("browser") is None: if _state.get("pw") is None: _state["pw"] = await async_playwright().start() _state["browser"] = await _state["pw"].chromium.launch() if _state.get("context") is None: _state["context"] = await _state["browser"].new_context() old = _state.get("page") if old is not None and not old.is_closed(): with suppress(Exception): await old.close() _state["page"] = await _state["context"].new_page() except PlaywrightError as e: # A missing browser binary surfaces here with playwright's own # "please run playwright install" text, which is kept verbatim: it # is the instruction the reader needs. await _shutdown() raise WebError(f"cannot start a browser: {e}") from None return _state["page"] def _store_shot(png: bytes, source: str) -> VisionResult: """A screenshot into the ImageStore, via the same path vision takes. Capped for the same reason: the viewport is 1280x720 by default but ``r.page.set_viewport_size`` is ambient, and a 3000px shot is bytes no vision model wants. Going through vision's own helpers means the key is content-addressed identically, so a screenshot dedupes against a webcam capture or a file read of the same image. """ import cv2 import numpy as np from .vision_tool import _cap_resolution, _encode, _store frame = cv2.imdecode(np.frombuffer(png, np.uint8), cv2.IMREAD_COLOR) if frame is None: raise WebError("the browser returned a screenshot cv2 cannot decode") frame = _cap_resolution(frame) height, width = frame.shape[:2] return _store(_encode(frame, "image/png"), "image/png", width, height, source) async def _run( url: str, wait_until: str, timeout: float, screenshot: bool ) -> PageResult: from playwright.async_api import Error as PlaywrightError if screenshot and image_store() is None: # Before launching anything: a browser with nowhere to put the # artifact is a wasted 0.3s and a confusing failure. raise WebError( "no ImageStore configured for this kernel (begin_cell got no" " images_dir), so a screenshot has nowhere to go — pass" " screenshot=False, or run under the execute server, which" " injects one" ) async with _state["lock"]: # One page, so concurrent runs are serialized rather than # interleaved: two gotos racing on one page would leave both results # describing whichever navigation landed last. gather() over run # still works, it just goes one at a time — and a model that wants # real tabs opens them itself (r.page.context.new_page()). page = await _browser_page() try: response = await page.goto( url, wait_until=wait_until, timeout=timeout * 1000 ) status = response.status if response is not None else 0 # A PageResult is a page that was RETRIEVED. fetch raises on # >= 400, so run raises too, or .status would mean two different # things in two modes of one tool — and a 404 body read as # content is a worse mistake than a caught error. Error bodies # stay reachable the ambient way: httpx, or playwright itself. if status >= 400: raise WebError(f"{status} {response.status_text} for {page.url}") content_type = ( response.headers.get("content-type", "") if response is not None else "" ) final = page.url html = await page.content() title = await page.title() png = await page.screenshot() if screenshot else None except WebError: raise except PlaywrightError as e: raise WebError(f"the browser failed on {url}: {e}") from None shot = ( await asyncio.to_thread(_store_shot, png, url) if png is not None else None ) # Outside the lock: extraction is the slow half (two subprocesses when # node runs) and it no longer touches the browser. The DOM is HTML # whatever the server claimed it was — that is the whole point of # rendering it — so the declared content type is not consulted. markdown, parsed_title, extracted = await asyncio.to_thread( _extract, html, "text/html" ) return PageResult( url=final, title=title or parsed_title, status=status, content_type=content_type, markdown=markdown, html=html, extracted=extracted, rendered=True, screenshot=shot, page=page, ) async def _close() -> BrowserClosed: async with _state["lock"]: return BrowserClosed(was_running=await _shutdown()) @subtool(tool="web") async def web( mode: str, target: str | None = None, limit: int | None = None, user_agent: str | None = None, wait_until: str | None = None, timeout: float | None = None, screenshot: bool | None = None, ): """Search the web, fetch a page from it, or render one in a browser. Args: mode: "search", "fetch", "run" or "close". target: search — the query. fetch/run — the URL, scheme included ("example.com" alone is an error, not a guess). close takes none. limit: search only — hits to keep (default 10). SearXNG returns dozens; ``.total`` says how many and ``.truncated`` whether the limit cut it. user_agent: fetch only — default "CrowAgent/1.0". Some sites block obvious bots, and some block everything that is not a browser; for those, mode="run" is the answer (a browser sends its own, and ``r.page.context`` is ambient if you need to change it). wait_until: run only — when the page counts as done: "load" (default), "domcontentloaded", "networkidle" (a SPA that renders after the load event), "commit". playwright's own vocabulary. timeout: run only — seconds to allow the navigation (default 30). Seconds here, not playwright's milliseconds. screenshot: run only — default True: a PNG of the viewport into the ImageStore, riding the row as an image block so a vision model SEES the page. Pass False in a loop over URLs, where one image per page is more context than the pages are worth. Returns: search -> WebSearchResult (.hits of WebHit(url, title, content, score, engines, published), .urls, .answers, .infoboxes, .suggestions, .corrections, .unresponsive, .total, .truncated); fetch/run -> PageResult (.markdown the whole extraction, .html the whole body — the RENDERED DOM for run — .title, .url final after redirects, .status, .content_type, .extracted, .rendered, .chars, .text windowed; run also .screenshot, a VisionResult, and .page, the live playwright Page); close -> BrowserClosed (.was_running). Raises: WebError: unknown mode, missing or blank target, an argument on the wrong mode, negative limit, non-positive timeout, unknown wait_until, SearXNG unreachable or non-200 or not JSON, DNS/TLS/timeout/missing-scheme on fetch, HTTP >= 400, a non-text content type (a PDF — the message says how to download it instead), a body over 10MB, a browser that cannot start (no chromium binary), a page that will not load, a screenshot with no ImageStore to put it in. Note: The model sees only what the cell PRINTS — ``print(r.text)`` after any mode. ``.markdown`` is the WHOLE page: slice it (``r.markdown[5000:9000]``) rather than paginating, and gather several searches (``asyncio.gather(*[web("search", q) for q in qs])``) rather than passing a list. Print ``.unresponsive`` too — engines down looks exactly like a bad query if you don't. fetch or run? fetch is httpx: fast, no browser, right for anything server-rendered. run is chromium: slower, and the only way to get what JavaScript builds or to be seen as a browser rather than a bot. ``.rendered`` on the result says which one happened. The browser is one chromium and one context for the kernel's lifetime, and ``.page`` is live in the next cell — click, type, wait_for_selector, then read the settled DOM with ``await r.page.content()``. The NEXT ``run`` closes it and opens a fresh page, so finish with a page before navigating again (cookies and storage survive; the context is the same). Everything playwright does is ambient; this tool starts the browser, navigates, captures and gets out of the way. ``web("close")`` shuts it all down. The client gets its own view regardless: one sibling call per row, titled with the query or the URL, carrying the rendered text and any screenshot. """ if mode not in _MODES: raise WebError( f"unknown web mode {mode!r} — expected one of: {', '.join(_MODES)}" ) if mode == "close": if target is not None: raise WebError( "mode='close' takes no target= — it closes the kernel's browser" ) return await _close() # Strip before the check: a blank query is a caller bug, and SearXNG # answers one with a plugin's clock reading instead of an error — # verified live, q=" " comes back with an answers=[{"engine": # "plugin: time_zone", "answer": "Sep 6, 2026, 8:24:44 AM"}] and no hits. # A nonsense success hides a mistake; an error names it. target = (target or "").strip() if not target: raise WebError( f"mode={mode!r} requires target=" + (" a query to search for" if mode == "search" else " a URL") ) # Cross-cutting, so no mode can quietly ignore an argument the caller # cared about. if limit is not None and mode != "search": raise WebError( f"limit= is for mode='search', not {mode!r} — a page is not" " paginated, slice r.markdown" ) if user_agent is not None and mode != "fetch": raise WebError(f"user_agent= is for mode='fetch', not {mode!r}") for name, value in ( ("wait_until", wait_until), ("timeout", timeout), ("screenshot", screenshot), ): if value is not None and mode != "run": raise WebError(f"{name}= is for mode='run', not {mode!r}") if mode == "search": return await _search(target, _cap(limit)) if mode == "fetch": return await _fetch(target, user_agent or _USER_AGENT) wait = wait_until or "load" if wait not in _WAIT_UNTIL: raise WebError( f"unknown wait_until {wait!r} — expected one of:" f" {', '.join(_WAIT_UNTIL)}" ) return await _run( target, wait, _seconds(timeout), True if screenshot is None else screenshot )