The Heading
Body one.
Body two.
"""web — search/fetch, Python-shaped: real result objects, WebError raised on failure, register + write-through like every subtool. No mocks and no internet. A local ThreadingHTTPServer stands in for BOTH the web (fetch routes) and SearXNG (a /search route serving canned JSON in the exact shape the real one returns, captured live) — so the real httpx client, the real streaming cap, the real readability/markdownify extraction and the real error paths all run, hermetically. SEARXNG_URL is read at call time, so pointing it at the fixture is the whole trick. Extraction assertions hold with OR without node: readabilipy's pure-Python mode produces an article too, so `extracted` is True either way and only the title's exact text differs. Nothing here may depend on node being installed. """ import asyncio import json import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import parse_qs, urlsplit import httpx import pytest from crow_cli.tools import web from crow_cli.tools.register import begin_cell, clear, pending from crow_cli.tools.results import ( BrowserClosed, PageResult, VisionResult, WebError, WebSearchResult, ) from crow_cli.tools.web_tool import _answer, _engine_down, _is_html, _is_texty MOD = sys.modules["crow_cli.tools.web_tool"] PAGE = ( "
Body one.
Body two.
" not in p.markdown assert p.html == PAGE # the raw body is kept whole, alongside assert p.chars == len(p.markdown) assert p.url == f"{base}/page.html" @pytest.mark.asyncio async def test_fetch_undeclared_html_is_sniffed(base): p = await web("fetch", f"{base}/html-noctype") assert p.content_type == "" and p.extracted and "Body one." in p.markdown @pytest.mark.asyncio async def test_fetch_undeclared_json_is_not_fed_to_readability(base): """The MCP tool's is_html treated a MISSING content type as HTML, so a JSON API that omitted the header was parsed as a page and came back mangled. A declared type is believed; only an undeclared body is sniffed, and only for ` 5000 assert p.text.startswith(f"{base}/huge.txt — 200 text/plain,") assert ", unextracted" in p.text.splitlines()[0] lines = p.text.splitlines() assert lines[-1] == f"… {p.chars - 5000:,} more chars — markdown[5000:]" assert len(p.text) < 5200 assert p.markdown[5000:5020] == HUGE.strip()[5000:5020] @pytest.mark.asyncio async def test_page_text_has_no_tail_when_the_page_fits(base): p = await web("fetch", f"{base}/plain.txt") assert "more chars" not in p.text assert p.text == ( f"{base}/plain.txt — 200 text/plain, 18 chars, unextracted\n" "just text\nline two" ) # --- run / close (a real chromium, skipped without one) --------------------- @pytest.mark.asyncio async def test_run_renders_what_fetch_cannot(chromium, base): """The distinction the mode exists for: fetch reads the bytes the server sent, run reads the DOM after the page's JavaScript ran. Same route, same request — different pages.""" f = await web("fetch", f"{base}/js") r = await web("run", f"{base}/js", screenshot=False) assert (f.title, f.markdown, f.rendered) == ("Static Title", "before", False) assert isinstance(r, PageResult) and r assert r.rendered and r.status == 200 assert r.title == "Rendered Title" assert r.markdown == "built by JS" assert "built by JS" in r.html assert r.url == f"{base}/js" assert ", rendered" in r.text.splitlines()[0] assert r.screenshot is None and "screenshot" not in r.text @pytest.mark.asyncio async def test_run_screenshot_rides_the_row(chromium, base, tmp_path): """The artifact that earns the wrapper its place: bytes on disk are invisible to the model, so the PNG goes to the ImageStore and its ref rides the row — the drain hydrates it into an image_url block.""" begin_cell(session_id="s1", parent_tool_call_id="t/c", images_dir=str(tmp_path / "img")) r = await web("run", f"{base}/js") assert isinstance(r.screenshot, VisionResult) assert r.screenshot.mime == "image/png" assert (r.screenshot.width, r.screenshot.height) == (1280, 720) assert r.screenshot.source == f"{base}/js" blobs = [p for p in (tmp_path / "img").rglob("*.png")] assert len(blobs) == 1 assert blobs[0].read_bytes()[:8] == b"\x89PNG\r\n\x1a\n" assert r.screenshot.image.size == (1280, 720) assert r.llm_images() == [{"key": r.screenshot.key, "mime": "image/png"}] entry = pending()[0] assert (entry.tool, entry.mode, entry.result_kind) == ("web", "run", "web") assert entry.llm_images == r.llm_images() assert ", screenshot" in entry.acp_payload["text"] @pytest.mark.asyncio async def test_run_screenshot_is_capped_like_vision(chromium, base, tmp_path, monkeypatch): """The viewport is 1280x720 by default but r.page.set_viewport_size is ambient, so a screenshot goes through vision's cap: a 3000px shot is bytes no vision model wants.""" vision_mod = sys.modules["crow_cli.tools.vision_tool"] monkeypatch.setattr(vision_mod, "_MAX_DIM", 100) begin_cell(session_id="s1", parent_tool_call_id="t/c", images_dir=str(tmp_path / "img")) r = await web("run", f"{base}/js") assert (r.screenshot.width, r.screenshot.height) == (100, 56) @pytest.mark.asyncio async def test_run_page_is_live_then_stale(chromium, base): """`.page` is raw playwright in the kernel — and the NEXT run closes it, so a stale one raises instead of silently pointing at a different navigation (verified live: a shared page made cell 1's r.page click the wrong thing by cell 3).""" first = await web("run", f"{base}/js", screenshot=False) page = first.page assert not page.is_closed() handle = await page.wait_for_selector("#app") assert await handle.text_content() == "built by JS" second = await web("run", f"{base}/page.html", screenshot=False) assert second.page is not page and page.is_closed() with pytest.raises(Exception, match="closed"): await page.title() @pytest.mark.asyncio async def test_run_context_survives_across_runs(chromium, base): """Cookies and storage live on the context, which is created once: a new page per run must not mean a new identity per run.""" await web("run", f"{base}/js", screenshot=False) first_context = MOD._state["context"] await web("run", f"{base}/page.html", screenshot=False) assert MOD._state["context"] is first_context @pytest.mark.asyncio async def test_run_gather_is_serialized_and_each_result_is_its_own(chromium, base): """One page at a time, so two gotos cannot race on it — but gather still returns one correct result per URL, each with its own page.""" rs = await asyncio.gather( *[web("run", f"{base}{u}", screenshot=False) for u in ("/js", "/page.html")] ) assert [r.title for r in rs] == ["Rendered Title", "Tiny Page"] assert rs[0].page is not rs[1].page assert rs[0].page.is_closed() and not rs[1].page.is_closed() @pytest.mark.asyncio async def test_run_http_error_raises_like_fetch(chromium, base): """A PageResult is a page that was RETRIEVED: fetch raises on >= 400 and so does run, or .status would mean two different things in two modes.""" with pytest.raises(WebError, match=rf"404 Not Found for {base}/missing"): await web("run", f"{base}/missing", screenshot=False) @pytest.mark.asyncio async def test_run_timeout_is_seconds(chromium, base): """The server sleeps 6s; 0.5s of patience must raise, in seconds, not in playwright's milliseconds.""" with pytest.raises(WebError, match=r"failed on .*hang"): await web("run", f"{base}/hang", screenshot=False, timeout=0.5) @pytest.mark.asyncio async def test_run_without_an_image_store(chromium, base, monkeypatch): """Checked BEFORE launching anything: a browser with nowhere to put the artifact is a wasted launch and a confusing failure, and the error says how to proceed either way.""" monkeypatch.setattr(MOD, "image_store", lambda: None) with pytest.raises(WebError, match="screenshot=False"): await web("run", f"{base}/js") r = await web("run", f"{base}/js", screenshot=False) assert r.screenshot is None @pytest.mark.asyncio async def test_close_is_idempotent_and_run_restarts(chromium, base): first = await web("close") assert isinstance(first, BrowserClosed) and not first.was_running assert first.text == "no browser was running" await web("run", f"{base}/js", screenshot=False) assert MOD._state["browser"] is not None closed = await web("close") assert closed.was_running and closed.text == "browser closed" assert MOD._state == {**MOD._state, "pw": None, "browser": None, "context": None, "page": None} again = await web("close") assert not again.was_running restarted = await web("run", f"{base}/js", screenshot=False) assert restarted.status == 200 @pytest.mark.asyncio async def test_close_records_a_row(): begin_cell(session_id="s1", parent_tool_call_id="t/c") await web("close") entry = pending()[0] assert (entry.tool, entry.mode, entry.status, entry.result_kind) == ( "web", "close", "completed", "text", ) assert entry.acp_payload["text"] == "no browser was running" # --- guards ---------------------------------------------------------------- @pytest.mark.asyncio async def test_unknown_mode(): with pytest.raises(WebError, match=r"unknown web mode 'crawl'"): await web("crawl", "x") @pytest.mark.asyncio async def test_missing_and_blank_target(): """A blank query is a caller bug, and SearXNG answers one with a plugin's clock reading instead of an error — a nonsense success that hides the mistake.""" for mode in ("search", "fetch"): with pytest.raises(WebError, match=rf"mode='{mode}' requires target="): await web(mode) with pytest.raises(WebError, match=rf"mode='{mode}' requires target="): await web(mode, " ") @pytest.mark.asyncio async def test_arguments_are_not_silently_ignored_on_the_wrong_mode(base): """Cross-cutting, checked before either mode runs: an argument the caller cared about must never be quietly dropped.""" with pytest.raises(WebError, match="a page is not paginated"): await web("fetch", f"{base}/plain.txt", limit=5) with pytest.raises(WebError, match="user_agent= is for mode='fetch'"): await web("search", "anything", user_agent="Bot/1") @pytest.mark.asyncio async def test_negative_limit(): with pytest.raises(WebError, match="limit must be >= 0, got -1"): await web("search", "anything", limit=-1) @pytest.mark.asyncio async def test_run_arguments_are_rejected_elsewhere(base): """wait_until/timeout/screenshot are browser vocabulary; passing them to fetch or search must not be silently dropped.""" with pytest.raises(WebError, match=r"screenshot= is for mode='run'"): await web("fetch", f"{base}/plain.txt", screenshot=True) with pytest.raises(WebError, match=r"wait_until= is for mode='run'"): await web("fetch", f"{base}/plain.txt", wait_until="load") with pytest.raises(WebError, match=r"timeout= is for mode='run'"): await web("search", "anything", timeout=5) @pytest.mark.asyncio async def test_wait_until_is_playwrights_vocabulary(): with pytest.raises(WebError, match=r"unknown wait_until 'whenever'"): await web("run", "https://example.com/", wait_until="whenever") @pytest.mark.asyncio async def test_timeout_must_be_positive(): for bad in (0, -1): with pytest.raises(WebError, match=rf"timeout must be > 0 seconds, got {bad}"): await web("run", "https://example.com/", timeout=bad) @pytest.mark.asyncio async def test_close_takes_no_target(): with pytest.raises(WebError, match=r"mode='close' takes no target="): await web("close", "https://example.com/") # --- internals ------------------------------------------------------------- def test_is_html_believes_a_declared_type(): assert _is_html("text/html; charset=utf-8", "") assert _is_html("application/xhtml+xml", "") assert not _is_html("application/json", "
x") assert not _is_html("text/plain", "") def test_is_html_sniffs_only_an_undeclared_body(): assert _is_html("", "") assert _is_html("", " ") assert not _is_html("", '{"a": 1}') assert not _is_html("", "") def test_is_texty(): assert _is_texty("text/plain") assert _is_texty("text/html; charset=utf-8") assert _is_texty("application/json") assert _is_texty("application/vnd.api+json") assert _is_texty("application/atom+xml") assert _is_texty("") # undeclared: assume text, then sniff assert not _is_texty("application/pdf") assert not _is_texty("image/png") assert not _is_texty("application/octet-stream") def test_answer_shapes(): assert _answer("plain") == "plain" assert _answer({"answer": "the text", "template": "t"}) == "the text" assert _answer({"engine": "e", "translations": [1]}) == '{"translations": [1]}' assert _answer(42) == "42" def test_engine_down_shapes(): assert _engine_down(["duckduckgo", "CAPTCHA"]) == "duckduckgo (CAPTCHA)" assert _engine_down(["duckduckgo"]) == "duckduckgo" assert _engine_down(["duckduckgo", ""]) == "duckduckgo" assert _engine_down("brave") == "brave" def test_use_readability_without_node(monkeypatch): """readabilipy's own have_node() spawns `node -v` per call and runs `npm install` when its node_modules is missing — and npm install chdirs the whole process. Availability is decided here instead, so a fetch can never trigger a network install or a cwd race in a kernel where other cells resolve relative paths.""" monkeypatch.setattr(MOD.shutil, "which", lambda _: None) assert MOD._use_readability() is False assert MOD._readability is False # cached: decided once per kernel def test_use_readability_without_bundled_node_modules(monkeypatch, tmp_path): """The other half of the guard: node present, readabilipy's bundled node_modules absent — exactly the state in which readabilipy would shell out to `npm install`. A stat of a directory decides it instead.""" import importlib.machinery import types monkeypatch.setattr(MOD.shutil, "which", lambda _: "/usr/bin/node") fake = types.ModuleType("readabilipy") fake.__spec__ = importlib.machinery.ModuleSpec( "readabilipy", None, origin=str(tmp_path / "readabilipy" / "__init__.py"), ) monkeypatch.setitem(sys.modules, "readabilipy", fake) assert MOD._use_readability() is False # --- register -------------------------------------------------------------- @pytest.mark.asyncio async def test_search_records_entry(): begin_cell(session_id="s1", parent_tool_call_id="t/c") await web("search", "anything") entries = pending() assert [(e.tool, e.mode, e.status, e.result_kind) for e in entries] == [ ("web", "search", "completed", "search") ] assert entries[0].args["target"] == "anything" payload = entries[0].acp_payload assert payload["content"] == "text" # A subject, not a path: a query is displayed, never claimed as a location. assert payload["subject"] == "anything" assert "path" not in payload assert payload["text"].startswith("anything — 3 of 3 hits") @pytest.mark.asyncio async def test_fetch_records_entry_with_the_url_as_subject(base): begin_cell(session_id="s1", parent_tool_call_id="t/c") await web("fetch", f"{base}/plain.txt") entry = pending()[0] assert (entry.tool, entry.mode, entry.status, entry.result_kind) == ( "web", "fetch", "completed", "web", ) assert entry.acp_payload["subject"] == f"{base}/plain.txt" @pytest.mark.asyncio async def test_failure_records_failed(base): begin_cell(session_id="s1", parent_tool_call_id="t/c") with pytest.raises(WebError): await web("fetch", f"{base}/missing") assert [(e.tool, e.mode, e.status) for e in pending()] == [ ("web", "fetch", "failed") ] assert "404" in pending()[0].error @pytest.mark.asyncio async def test_gather_records_one_row_per_call(): """One row is one call — the model's parallelism must not collapse into a single ACP tool call. Rows land in COMPLETION order, which is the honest order for concurrent calls (and each carries started_at/ended_at), so this compares as a set rather than a sequence.""" begin_cell(session_id="s1", parent_tool_call_id="t/c") await asyncio.gather(*[web("search", q) for q in ("one", "two", "three")]) assert sorted(e.args["target"] for e in pending()) == ["one", "three", "two"] assert all(e.parent_tool_call_id == "t/c" for e in pending())