diff --git a/README.md b/README.md index ba47360..b8c0730 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,16 @@ if r.http_status >= 400: raise SystemExit(f"{r.url} is an error page, not content") ``` +`find_element` and `interactive_elements` return elements that act on +themselves: each is still the dict the browser sent, and every action that +takes a `backend_node_id` is also a method on it: + +```python +found = page.find_element(role="button", name="Accept") +if found: + found[0].click() # also fill(value=...), hover(), markdown(), ... +``` + ## Examples [`examples/`](examples/) scrapes a JavaScript-rendered site that `requests` diff --git a/lightpanda/__init__.py b/lightpanda/__init__.py index f3c4b7e..9e89541 100644 --- a/lightpanda/__init__.py +++ b/lightpanda/__init__.py @@ -36,19 +36,21 @@ __docformat__ = "google" -from .async_browser import AsyncBrowser, AsyncSession, run_script_async +from .async_browser import AsyncBrowser, AsyncElement, AsyncSession, run_script_async from .webdriver import AsyncWebDriverServer, WebDriverServer -from .browser import Browser, PageResult, Session, run_script +from .browser import Browser, Element, PageResult, Session, run_script from .cdp import AsyncCDPServer, CDPServer from .errors import LightpandaError, ProcessError, ProtocolError, ScriptError, ToolError __all__ = [ "Browser", "Session", + "Element", "PageResult", "run_script", "AsyncBrowser", "AsyncSession", + "AsyncElement", "run_script_async", "CDPServer", "AsyncCDPServer", diff --git a/lightpanda/_methods.py b/lightpanda/_methods.py index 03b2b78..85229d3 100644 --- a/lightpanda/_methods.py +++ b/lightpanda/_methods.py @@ -2,8 +2,9 @@ One method per lightpanda browser tool, signatures derived from the tool JSON schemas, with tool and parameter names in snake_case (the schemas' -camelCase names still work through ``Session.call``). ``AsyncSessionMethods`` -is the same surface with ``async def`` signatures. +camelCase names still work through ``Session.call``). ``ElementMethods`` +holds the tools that take a ``backendNodeId``, aimed at the element itself. +The ``Async`` classes are the same surfaces with ``async def`` signatures. """ from __future__ import annotations @@ -11,7 +12,8 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from .browser import PageResult + from .async_browser import AsyncElement + from .browser import Element, PageResult class SessionMethods: @@ -56,21 +58,25 @@ def evaluate(self, *, script: str, url: str | None = None, timeout: int | None = return self.call("evaluate", script=script, url=url, timeout=timeout, save=save) def extract(self, *, schema: str | dict | list, save: str | None = None) -> Any: """Extract structured data from the current page (navigate first). `schema` is a JSON object (passed as a string) mapping output field names to CSS-selector specs. It is NOT a JSON Schema — no "type"/"properties" wrappers; the keys ARE your output fields. Value shapes: - "" → first match's text (trimmed; null if no match) - [""] → every match's text (string[]) - {"selector":"","attr":""} → first match's attribute value (href/src resolved to absolute URLs) - [{"selector":"","attr":""}] → every match's attribute (string[]) - [{"selector":"","fields":{…}}] → one object per match; field selectors resolve relative to that match and accept any shape above ("" = the match's own text; nest arrays for per-item sub-lists) + ```text + "" → first match's text (trimmed; null if no match) + [""] → every match's text (string[]) + {"selector":"","attr":""} → first match's attribute value (href/src resolved to absolute URLs) + [{"selector":"","attr":""}] → every match's attribute (string[]) + [{"selector":"","fields":{…}}] → one object per match; field selectors resolve relative to that match and accept any shape above ("" = the match's own text; nest arrays for per-item sub-lists) + ``` Add "limit": N inside any array's object spec to cap matches. Every extracted value is a string or null — parse numbers downstream. An empty array is a valid result, but if ALL top-level keys miss, the call errors: inspect the page (tree/markdown) and retry with corrected selectors. Finish data tasks with extract — it is the only read recorded as a replayable `extract(...)` script call; answers lifted from `markdown` text in chat are not. Examples (schema → result): - {"karma": "#karma"} → {"karma":"42"} - {"items": [".story .title"]} → {"items":["Title 1","Title 2"]} - {"top3": [{"selector":".story .title","limit":3}]} → {"top3":["A","B","C"]} - {"links": [{"selector":"a.title","attr":"href"}]} → {"links":["https://site/a","https://site/b"]} - {"stories": [{"selector":".athing","fields":{"title":".titleline","rank":".rank"}}]} → {"stories":[{"title":"Foo","rank":"1"}]} + ```text + {"karma": "#karma"} → {"karma":"42"} + {"items": [".story .title"]} → {"items":["Title 1","Title 2"]} + {"top3": [{"selector":".story .title","limit":3}]} → {"top3":["A","B","C"]} + {"links": [{"selector":"a.title","attr":"href"}]} → {"links":["https://site/a","https://site/b"]} + {"stories": [{"selector":".athing","fields":{"title":".titleline","rank":".rank"}}]} → {"stories":[{"title":"Foo","rank":"1"}]} + ``` Args: schema: Extraction schema as a string: a JSON object literal mapping output field names to CSS-selector specs (see tool description). Not a JSON Schema. @@ -93,7 +99,7 @@ def fill(self, *, value: str, selector: str | None = None, backend_node_id: int - `title`: Document title. Empty or absent when the page has none. """ return self.call("fill", value=value, selector=selector, backend_node_id=backend_node_id) - def find_element(self, *, role: str | None = None, name: str | None = None) -> Any: + def find_element(self, *, role: str | None = None, name: str | None = None) -> list[Element]: """Find interactive elements by role and/or accessible name. Returns matching elements with their backend node IDs. Useful for locating specific elements without parsing the full semantic tree. Args: @@ -162,7 +168,7 @@ def html(self, *, selector: str | None = None, backend_node_id: int | None = Non timeout: Optional timeout in milliseconds. Defaults to 10000. """ return self.call("html", selector=selector, backend_node_id=backend_node_id, max_bytes=max_bytes, strip=strip, url=url, timeout=timeout) - def interactive_elements(self, *, url: str | None = None, timeout: int | None = None) -> Any: + def interactive_elements(self, *, url: str | None = None, timeout: int | None = None) -> list[Element]: """List every visible interactive element on the page as a JSON array: native controls, ARIA widgets, contenteditable regions, elements with event listeners, and focusable elements. Each entry has `backendNodeId`, `tagName`, `role`, `name`, `type` (why it counts as interactive), `tabIndex`, and when present `listeners`, `disabled`, `id`, `class`, `href`, `inputType`, `value`, `elementName` and `placeholder`. Use it to survey what can be acted on; to locate one element by role or name, `findElement` is cheaper. If a url is provided, it navigates there first. Args: @@ -292,7 +298,7 @@ def structured_data(self, *, url: str | None = None, timeout: int | None = None) timeout: Optional timeout in milliseconds. Defaults to 10000. """ return self.call("structuredData", url=url, timeout=timeout) - def tree(self, *, url: str | None = None, timeout: int | None = None, backend_node_id: int | None = None, max_depth: int | None = None) -> Any: + def tree(self, *, url: str | None = None, timeout: int | None = None, backend_node_id: int | None = None, max_depth: int | None = None, max_bytes: int | None = None) -> Any: """Semantic outline of the page as indented text: one node per line with its role, accessible name, value and backendNodeId, plus checked state and select options with the selected one marked. The default first read of an unfamiliar page; input and select values are already here, so no `nodeDetails` call is needed to read them. Pass `backendNodeId` to scope to a subtree and `maxDepth` to survey structure before going deeper. Read it again after any page-changing action, since the DOM it describes may have changed; use `nodeDetails` to turn a backendNodeId into a CSS selector for actions. Args: @@ -300,8 +306,9 @@ def tree(self, *, url: str | None = None, timeout: int | None = None, backend_no timeout: Optional timeout in milliseconds. Defaults to 10000. backend_node_id: Optional backend node ID to get the tree for a specific element instead of the document root. 0 is treated as omitted. max_depth: Optional maximum depth of the tree to return. Useful for exploring high-level structure first. + max_bytes: Optional cap on output size in bytes. Past it, lines are kept by priority (alerts, dialogs, invalid fields and the focused element, then controls and named nodes, then text; list entries breadth-first), and dropped lines are counted in a '[+N more @id]' marker. id is the backendNodeId of the node holding them, often a list item or list that isn't printed itself: call tree with backendNodeId=id to read them. """ - return self.call("tree", url=url, timeout=timeout, backend_node_id=backend_node_id, max_depth=max_depth) + return self.call("tree", url=url, timeout=timeout, backend_node_id=backend_node_id, max_depth=max_depth, max_bytes=max_bytes) def wait_for_script(self, *, script: str, timeout: int | None = None) -> Any: """Wait until a JS expression returns truthy. Re-evaluates on each tick of the event loop. Use for synchronization beyond what CSS selectors can express — e.g. `window.dataLoaded === true`, `document.readyState === 'complete'`, `document.querySelectorAll('.row').length >= 5`. @@ -327,6 +334,140 @@ def wait_for_state(self, *, state: str, timeout: int | None = None) -> Any: """ return self.call("waitForState", state=state, timeout=timeout) +class ElementMethods: + def call(self, tool: str, **kwargs: Any) -> Any: + raise NotImplementedError + + def click(self) -> PageResult: + """Click on an interactive element. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Returns the current page URL and title after the click. + + Returns: + PageResult: The sentence above, with these attributes (None when absent): + + - `url`: URL of the page the call left loaded. + - `http_status`: Response status of that page's own document. Absent when no response has arrived. 4xx/5xx means an error page, not the content. + - `title`: Document title. Empty or absent when the page has none. + """ + return self.call("click") + def fill(self, *, value: str) -> PageResult: + """Fill text into an input element. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. + + Args: + value: The text to fill into the input element. + + Returns: + PageResult: The sentence above, with these attributes (None when absent): + + - `url`: URL of the page the call left loaded. + - `http_status`: Response status of that page's own document. Absent when no response has arrived. 4xx/5xx means an error page, not the content. + - `title`: Document title. Empty or absent when the page has none. + """ + return self.call("fill", value=value) + def hover(self) -> PageResult: + """Hover over an element, triggering mouseover and mouseenter events. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Useful for menus, tooltips, and hover states. + + Returns: + PageResult: The sentence above, with these attributes (None when absent): + + - `url`: URL of the page the call left loaded. + - `http_status`: Response status of that page's own document. Absent when no response has arrived. 4xx/5xx means an error page, not the content. + - `title`: Document title. Empty or absent when the page has none. + """ + return self.call("hover") + def html(self, *, max_bytes: int | None = None, strip: dict | None = None) -> Any: + """Raw HTML for the document or, with `selector`/`backendNodeId`, a single node's outerHTML. Verbose; use only when you need attributes that markdown discards. + + Args: + max_bytes: Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap. + strip: Optional. Omit element groups from the output: `js` (script, noscript, script preloads), `css` (style, stylesheet links), `ui` (css plus img, picture, video, audio, svg, canvas, iframe), `invisible` (elements an author rule or inline style sets to display:none), `shell` (nav, aside, dialog, page-level header/footer and the matching landmark roles; skipped when that would drop most of the text), `clutter` (keep only the main content, in the manner of reader modes; includes `shell` and `invisible`, and falls back to `shell` when it finds too little). {"js":true,"css":true} keeps a page dump small. + """ + return self.call("html", max_bytes=max_bytes, strip=strip) + def markdown(self, *, max_bytes: int | None = None, strip: dict | None = None) -> Any: + """Render the page (or a subtree) as markdown. Scope with `selector` or `backendNodeId` to read just the relevant region — full-page markdown is the last resort. Use `maxBytes` to cap long pages. + + Args: + max_bytes: Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap. + strip: Optional. Omit element groups from the output; same groups as the html tool's strip. `shell` (page chrome by markup) and `clutter` (keep only the main content, in the manner of reader modes) are the ones that matter for reading; `ui` also drops images. + """ + return self.call("markdown", max_bytes=max_bytes, strip=strip) + def node_details(self) -> Any: + """Details for a node by backendNodeId: a ready-to-use CSS `selector` that resolves to the node (the first match, as click/fill resolve it), plus tag, role, name, interactivity, disabled, value, input type, placeholder, href, id, class, checked, select options. The canonical way to turn a tree backendNodeId into a CSS selector.""" + return self.call("nodeDetails") + def press(self, *, key: str) -> PageResult: + """Press a keyboard key, dispatching keydown and keyup events. Use key names like 'Enter', 'Tab', 'Escape', 'ArrowDown', 'Backspace', or single characters like 'a', '1'. Common shorthand is normalized: 'enter'/'return' → 'Enter', 'esc' → 'Escape', 'up'/'down'/'left'/'right' → 'Arrow*', 'space' → ' '. Pressing 'Enter' on a form input or submit button triggers implicit form submission. + + Args: + key: The key to press (e.g. 'Enter', 'Tab', 'a'). + + Returns: + PageResult: The sentence above, with these attributes (None when absent): + + - `url`: URL of the page the call left loaded. + - `http_status`: Response status of that page's own document. Absent when no response has arrived. 4xx/5xx means an error page, not the content. + - `title`: Document title. Empty or absent when the page has none. + """ + return self.call("press", key=key) + def screenshot(self, *, path: str | None = None, full_page: bool | None = None, strip: dict | None = None) -> Any: + """Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file at full size and returns its location; without it, returns the image inline where the client can display one, at most 1280px wide and 4096px tall. Use it to see spatial layout; read content with `markdown`/`tree`. + + Args: + path: Optional relative path (no '..' segments) to write the PNG to. Created or overwritten. Without it the image is returned inline, which needs a client that can display images. + full_page: Render the whole content height instead of one viewport. Defaults to false. + strip: Optional. Omit element groups from the render; same groups as the html tool's strip (`js`, `css`, `ui`, `invisible`, `shell`, `clutter`). + """ + return self.call("screenshot", path=path, full_page=full_page, strip=strip) + def scroll(self, *, x: int | None = None, y: int | None = None) -> PageResult: + """Scroll the window, or an element's scroll container, to an absolute position; an omitted axis keeps its current offset. Target an element with a CSS selector (preferred for reproducibility) or a backendNodeId; omit both to scroll the window. Page scripts receive a `scroll` event, so content that loads on scroll (infinite feeds, lazy lists) may appear: read the page again afterwards, with `waitForState` if it is still loading. Returns the final scroll position and the current page URL and title. + + Args: + x: Optional: The horizontal scroll offset. + y: Optional: The vertical scroll offset. + + Returns: + PageResult: The sentence above, with these attributes (None when absent): + + - `url`: URL of the page the call left loaded. + - `http_status`: Response status of that page's own document. Absent when no response has arrived. 4xx/5xx means an error page, not the content. + - `title`: Document title. Empty or absent when the page has none. + """ + return self.call("scroll", x=x, y=y) + def select_option(self, *, value: str) -> PageResult: + """Select an option in a dropdown element by its value. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Dispatches input and change events. + + Args: + value: The value of the option to select. + + Returns: + PageResult: The sentence above, with these attributes (None when absent): + + - `url`: URL of the page the call left loaded. + - `http_status`: Response status of that page's own document. Absent when no response has arrived. 4xx/5xx means an error page, not the content. + - `title`: Document title. Empty or absent when the page has none. + """ + return await self.call("selectOption", value=value) + async def set_checked(self, *, checked: bool) -> PageResult: + """Check or uncheck a checkbox or radio button. Provide either a CSS selector (preferred for reproducibility) or a backendNodeId. Dispatches input, change, and click events. + + Args: + checked: Whether to check (true) or uncheck (false) the element. + + Returns: + PageResult: The sentence above, with these attributes (None when absent): + + - `url`: URL of the page the call left loaded. + - `http_status`: Response status of that page's own document. Absent when no response has arrived. 4xx/5xx means an error page, not the content. + - `title`: Document title. Empty or absent when the page has none. + """ + return await self.call("setChecked", checked=checked) + async def tree(self, *, max_depth: int | None = None, max_bytes: int | None = None) -> Any: + """Semantic outline of the page as indented text: one node per line with its role, accessible name, value and backendNodeId, plus checked state and select options with the selected one marked. The default first read of an unfamiliar page; input and select values are already here, so no `nodeDetails` call is needed to read them. Pass `backendNodeId` to scope to a subtree and `maxDepth` to survey structure before going deeper. Read it again after any page-changing action, since the DOM it describes may have changed; use `nodeDetails` to turn a backendNodeId into a CSS selector for actions. + + Args: + max_depth: Optional maximum depth of the tree to return. Useful for exploring high-level structure first. + max_bytes: Optional cap on output size in bytes. Past it, lines are kept by priority (alerts, dialogs, invalid fields and the focused element, then controls and named nodes, then text; list entries breadth-first), and dropped lines are counted in a '[+N more @id]' marker. id is the backendNodeId of the node holding them, often a list item or list that isn't printed itself: call tree with backendNodeId=id to read them. + """ + return await self.call("tree", max_depth=max_depth, max_bytes=max_bytes) diff --git a/lightpanda/async_browser.py b/lightpanda/async_browser.py index e3de672..b1eb5cd 100644 --- a/lightpanda/async_browser.py +++ b/lightpanda/async_browser.py @@ -17,11 +17,12 @@ from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor -from .browser import Browser, Session, _generated, run_script +from .browser import Browser, Element, Session, _generated, run_script from .client import _documented from .errors import LightpandaError AsyncSessionMethods = _generated("AsyncSessionMethods") +AsyncElementMethods = _generated("AsyncElementMethods") async def _run(executor: ThreadPoolExecutor, fn, *args, **kwargs): @@ -52,7 +53,10 @@ def id(self) -> str: async def call(self, tool: str, **kwargs): """Invoke a browser tool by name. The generated methods route here. Same contract as :meth:`Session.call`, awaitable.""" - return await _run(self._executor, self._session.call, tool, **kwargs) + result = await _run(self._executor, self._session.call, tool, **kwargs) + if isinstance(result, list): + return [AsyncElement(node, self) if isinstance(node, Element) else node for node in result] + return result def __getattr__(self, attr: str): session = self.__dict__.get("_session") @@ -72,6 +76,34 @@ async def __aexit__(self, *exc): +@_documented +class AsyncElement(AsyncElementMethods, dict): + """A node :meth:`AsyncSession.find_element` or + :meth:`AsyncSession.interactive_elements` found, which acts on itself. + + Same as :class:`Element`, with every method a coroutine to await: + + ```python + found = await page.find_element(role="button", name="Accept") + if found: + await found[0].click() + ``` + """ + + def __init__(self, fields: dict, session: AsyncSession): + super().__init__(fields) + self._session = session + + @property + def backend_node_id(self) -> int: + """The node's id, as :meth:`AsyncSession.click` and the others take it.""" + return self["backendNodeId"] + + async def call(self, tool: str, **kwargs): + """Invoke a browser tool on this node; see :meth:`AsyncSession.call`.""" + return await self._session.call(tool, backendNodeId=self.backend_node_id, **kwargs) + + class AsyncBrowser: """A lightpanda browser process, driven from asyncio. diff --git a/lightpanda/browser.py b/lightpanda/browser.py index 3b71012..37c895b 100644 --- a/lightpanda/browser.py +++ b/lightpanda/browser.py @@ -73,6 +73,45 @@ def _generated(name: str) -> type: SessionMethods = _generated("SessionMethods") +ElementMethods = _generated("ElementMethods") + +# Tools answering with a list of nodes, which `Session.call` returns as Elements. +_ELEMENT_TOOLS = {"findElement", "interactiveElements"} +# Element methods aim at the element itself: no other target, no navigation. +_ELEMENT_PARAMS = {"selector", "backendNodeId", "url", "timeout"} + + +@_documented +class Element(ElementMethods, dict): + """A node :meth:`Session.find_element` or :meth:`Session.interactive_elements` + found, which acts on itself. + + Still the dict the browser sent, so ``element["name"]`` reads its fields, + with every action that takes a ``backend_node_id`` as a method aimed at + this node: + + ```python + found = page.find_element(role="button", name="Accept") + if found: + found[0].click() + ``` + + The element belongs to its page: once the page navigates away, its node + is gone and the methods fail. + """ + + def __init__(self, fields: dict, session: Session): + super().__init__(fields) + self._session = session + + @property + def backend_node_id(self) -> int: + """The node's id, as :meth:`Session.click` and the others take it.""" + return self["backendNodeId"] + + def call(self, tool: str, **kwargs): + """Invoke a browser tool on this node; see :meth:`Session.call`.""" + return self._session.call(tool, backendNodeId=self.backend_node_id, **kwargs) @_documented @@ -167,9 +206,12 @@ def call(self, tool: str, **kwargs): if page_state is not None: return PageResult(text, page_state) try: - return json.loads(text) + parsed = json.loads(text) except ValueError: return text + if name in _ELEMENT_TOOLS and isinstance(parsed, list): + return [Element(node, self) for node in parsed] + return parsed def _resolve(self, attr: str) -> str | None: """The tool name behind a snake_case public attribute, if any.""" diff --git a/scripts/generate_methods.py b/scripts/generate_methods.py index 65770e8..81a584e 100644 --- a/scripts/generate_methods.py +++ b/scripts/generate_methods.py @@ -8,7 +8,9 @@ declare an ``outputSchema`` answer with ``structuredContent``, which ``Session.call`` returns as a ``PageResult``, so those methods are annotated with it and get a ``Returns:`` section listing its attributes from the -schema's property descriptions; the rest stay ``Any``. Being real code, the methods are +schema's property descriptions; the rest stay ``Any``. Tools that take a +``backendNodeId`` also get a method on ``ElementMethods``, aimed at the element +it is called on. Being real code, the methods are visible to IDEs, type checkers, and pdoc alike. Run with a binary available: uv run --no-project python scripts/generate_methods.py @@ -25,7 +27,13 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from lightpanda import Browser # noqa: E402 -from lightpanda.browser import _SESSION_TOOLS, PageResult, _snake # noqa: E402 +from lightpanda.browser import ( # noqa: E402 + _ELEMENT_PARAMS, + _ELEMENT_TOOLS, + _SESSION_TOOLS, + PageResult, + _snake, +) PY_TYPES = {"string": "str", "integer": "int", "number": "float", "boolean": "bool", "object": "dict", "array": "list"} @@ -34,8 +42,9 @@ One method per lightpanda browser tool, signatures derived from the tool JSON schemas, with tool and parameter names in snake_case (the schemas' -camelCase names still work through ``Session.call``). ``AsyncSessionMethods`` -is the same surface with ``async def`` signatures. +camelCase names still work through ``Session.call``). ``ElementMethods`` +holds the tools that take a ``backendNodeId``, aimed at the element itself. +The ``Async`` classes are the same surfaces with ``async def`` signatures. """ from __future__ import annotations @@ -43,7 +52,8 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from .browser import PageResult + from .async_browser import AsyncElement + from .browser import Element, PageResult ''' CLASS_HEADER = '''\ @@ -85,9 +95,11 @@ def docstring(description: str, args: list[tuple[str, str]], returns: str = "") return f' """{indented.lstrip()}\n """' -def method_source(name: str, spec: dict, is_async: bool = False) -> str: +def method_source(name: str, spec: dict, is_async: bool = False, on_element: bool = False) -> str: schema = spec["schema"] properties = schema.get("properties", {}) + if on_element: + properties = {prop: v for prop, v in properties.items() if prop not in _ELEMENT_PARAMS} required = set(schema.get("required", [])) args = {prop: _snake(prop) for prop in properties} for prop, arg in args.items(): @@ -119,6 +131,8 @@ def method_source(name: str, spec: dict, is_async: bool = False) -> str: await_ = "await " if is_async else "" output_schema = spec.get("output_schema") returns = "PageResult" if output_schema else "Any" + if name in _ELEMENT_TOOLS: + returns = "list[AsyncElement]" if is_async else "list[Element]" lines = [f" {prefix} {snake}({', '.join(params)}) -> {returns}:"] section = returns_section(name, output_schema) if output_schema else "" lines.append(docstring(spec["description"], documented, section)) @@ -131,16 +145,22 @@ def main() -> None: with Browser() as browser: tools = {k: v for k, v in browser.tools.items() if k not in _SESSION_TOOLS} + on_node = sorted(name for name in tools if "backendNodeId" in tools[name]["schema"].get("properties", {})) + # Elements are dicts: a tool must not shadow one of dict's methods. + if shadowed := [name for name in on_node if hasattr(dict, _snake(name))]: + raise SystemExit(f"tools {shadowed} would shadow dict methods on Element") blocks = [HEADER] for is_async in (False, True): - cls = "AsyncSessionMethods" if is_async else "SessionMethods" - blocks.append(CLASS_HEADER.format(cls=cls, prefix="async " if is_async else "")) + prefix = "Async" if is_async else "" + blocks.append(CLASS_HEADER.format(cls=f"{prefix}SessionMethods", prefix="async " if is_async else "")) blocks += [method_source(name, tools[name], is_async) for name in sorted(tools)] + blocks.append(CLASS_HEADER.format(cls=f"{prefix}ElementMethods", prefix="async " if is_async else "")) + blocks += [method_source(name, tools[name], is_async, on_element=True) for name in on_node] source = "\n".join(blocks) + "\n" compile(source, str(out), "exec") out.write_text(source) - print(f"wrote {out} ({len(tools)} tools)") + print(f"wrote {out} ({len(tools)} tools, {len(on_node)} on elements)") if __name__ == "__main__": diff --git a/tests/test_async.py b/tests/test_async.py index f5452f7..970cf7b 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -6,7 +6,13 @@ import pytest -from lightpanda import AsyncBrowser, LightpandaError, ToolError, run_script_async +from lightpanda import ( + AsyncBrowser, + AsyncElement, + LightpandaError, + ToolError, + run_script_async, +) async def test_goto_markdown_and_snake_case(abrowser, fixture_url): @@ -28,6 +34,15 @@ async def visit(path): assert [u.rsplit("/", 1)[1] for u in urls] == ["index.html", "other.html", "index.html"] +async def test_found_elements_act_on_themselves(abrowser, fixture_url): + async with abrowser.session() as page: + await page.goto(url=f"{fixture_url}/index.html") + found = await page.find_element(role="link", name="Second") + assert [type(el) for el in found] == [AsyncElement] + assert found[0]["name"] == "Second item" + assert (await found[0].click()).url == f"{fixture_url}/other.html" + + async def test_tool_error_raises(abrowser, fixture_url): async with abrowser.session() as page: await page.goto(url=f"{fixture_url}/index.html") diff --git a/tests/test_browser.py b/tests/test_browser.py index 15f0231..232aa53 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -1,3 +1,4 @@ +import json import os import signal import subprocess @@ -8,7 +9,7 @@ import pytest from conftest import alive -from lightpanda import Browser, LightpandaError, PageResult, ScriptError, ToolError, run_script, client +from lightpanda import Browser, Element, LightpandaError, PageResult, ScriptError, ToolError, run_script, client def test_goto_and_markdown(browser, fixture_url): @@ -45,6 +46,24 @@ def test_page_result_carries_status(browser, fixture_url): assert clicked.http_status == 200 +def test_found_elements_act_on_themselves(browser, fixture_url): + with browser.new_session() as page: + page.goto(url=f"{fixture_url}/index.html") + found = page.find_element(role="link", name="Second") + assert [type(el) for el in found] == [Element] + link = found[0] + # Still the dict the browser sent. + assert link["name"] == "Second item" + assert json.loads(json.dumps(link)) == dict(link) + assert link.backend_node_id == link["backendNodeId"] + + assert link.markdown().strip() == f"[Second item]({fixture_url}/other.html)" + assert link.click().url == f"{fixture_url}/other.html" + + page.goto(url=f"{fixture_url}/index.html") + assert all(isinstance(el, Element) for el in page.interactive_elements()) + + def test_extract_dict_schema(browser, fixture_url): with browser.new_session() as page: page.goto(url=f"{fixture_url}/index.html")