A node responsible for fetching the HTML content of a specified URL and all its sub-links recursively up to a certain level of hyperlink the graph. This content is then used to update the graph's state. It uses ChromiumLoader to fetch the content from a web page asynchronously (with
| 13 | |
| 14 | |
| 15 | class FetchNodeLevelK(BaseNode): |
| 16 | """ |
| 17 | A node responsible for fetching the HTML content of a specified URL and all its sub-links |
| 18 | recursively up to a certain level of hyperlink the graph. This content is then used to update |
| 19 | the graph's state. It uses ChromiumLoader to fetch the content from a web page asynchronously |
| 20 | (with proxy protection). |
| 21 | |
| 22 | Attributes: |
| 23 | embedder_model: An optional model for embedding the fetched content. |
| 24 | verbose (bool): A flag indicating whether to show print statements during execution. |
| 25 | cache_path (str): Path to cache fetched content. |
| 26 | headless (bool): Whether to run the Chromium browser in headless mode. |
| 27 | loader_kwargs (dict): Additional arguments for the content loader. |
| 28 | browser_base (dict): Optional configuration for the browser base API. |
| 29 | depth (int): Maximum depth of hyperlink graph traversal. |
| 30 | only_inside_links (bool): Whether to fetch only internal links. |
| 31 | min_input_len (int): Minimum required length of input data. |
| 32 | |
| 33 | Args: |
| 34 | input (str): Boolean expression defining the input keys needed from the state. |
| 35 | output (List[str]): List of output keys to be updated in the state. |
| 36 | node_config (dict): Additional configuration for the node. |
| 37 | node_name (str): The unique identifier name for the node, defaulting to "FetchLevelK". |
| 38 | """ |
| 39 | |
| 40 | def __init__( |
| 41 | self, |
| 42 | input: str, |
| 43 | output: List[str], |
| 44 | node_config: Optional[dict] = None, |
| 45 | node_name: str = "FetchLevelK", |
| 46 | ): |
| 47 | """ |
| 48 | Initializes the FetchNodeLevelK instance. |
| 49 | |
| 50 | Args: |
| 51 | input (str): Boolean expression defining the input keys needed from the state. |
| 52 | output (List[str]): List of output keys to be updated in the state. |
| 53 | node_config (Optional[dict]): Additional configuration for the node. |
| 54 | node_name (str): The name of the node (default is "FetchLevelK"). |
| 55 | """ |
| 56 | super().__init__(node_name, "node", input, output, 2, node_config) |
| 57 | |
| 58 | self.embedder_model = node_config.get("embedder_model", None) |
| 59 | self.verbose = node_config.get("verbose", False) if node_config else False |
| 60 | self.cache_path = node_config.get("cache_path", False) |
| 61 | self.headless = node_config.get("headless", True) if node_config else True |
| 62 | self.loader_kwargs = node_config.get("loader_kwargs", {}) if node_config else {} |
| 63 | self.browser_base = node_config.get("browser_base", None) |
| 64 | self.scrape_do = node_config.get("scrape_do", None) |
| 65 | self.storage_state = node_config.get("storage_state", None) |
| 66 | self.depth = node_config.get("depth", 1) if node_config else 1 |
| 67 | self.only_inside_links = ( |
| 68 | node_config.get("only_inside_links", False) if node_config else False |
| 69 | ) |
| 70 | self.min_input_len = 1 |
| 71 | |
| 72 | def execute(self, state: dict) -> dict: |