Source code for langchain.chains.natbot.crawler
# flake8: noqa
import time
from sys import platform
from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    Iterable,
    List,
    Optional,
    Set,
    Tuple,
    TypedDict,
    Union,
)
if TYPE_CHECKING:
    from playwright.sync_api import Browser, CDPSession, Page, sync_playwright
black_listed_elements: Set[str] = {
    "html",
    "head",
    "title",
    "meta",
    "iframe",
    "body",
    "script",
    "style",
    "path",
    "svg",
    "br",
    "::marker",
}
[docs]class ElementInViewPort(TypedDict):
    """A typed dictionary containing information about elements in the viewport."""
    node_index: str
    backend_node_id: int
    node_name: Optional[str]
    node_value: Optional[str]
    node_meta: List[str]
    is_clickable: bool
    origin_x: int
    origin_y: int
    center_x: int
    center_y: int
[docs]class Crawler:
    """A crawler for web pages.
    **Security Note**: This is an implementation of a crawler that uses a browser via
        Playwright.
        This crawler can be used to load arbitrary webpages INCLUDING content
        from the local file system.
        Control access to who can submit crawling requests and what network access
        the crawler has.
        Make sure to scope permissions to the minimal permissions necessary for
        the application.
        See https://python.langchain.com/docs/security for more information.
    """
[docs]    def __init__(self) -> None: