A node that can filter out the relevant links in the webpage content for the user prompt. Node expects the already scrapped links on the webpage and hence it is expected that this node be used after the FetchNode. Attributes: llm_model: An instance of the language model cli
| 16 | |
| 17 | |
| 18 | class SearchLinkNode(BaseNode): |
| 19 | """ |
| 20 | A node that can filter out the relevant links in the webpage content for the user prompt. |
| 21 | Node expects the already scrapped links on the webpage and hence it is expected |
| 22 | that this node be used after the FetchNode. |
| 23 | |
| 24 | Attributes: |
| 25 | llm_model: An instance of the language model client used for generating answers. |
| 26 | verbose (bool): A flag indicating whether to show print statements during execution. |
| 27 | |
| 28 | Args: |
| 29 | input (str): Boolean expression defining the input keys needed from the state. |
| 30 | output (List[str]): List of output keys to be updated in the state. |
| 31 | node_config (dict): Additional configuration for the node. |
| 32 | node_name (str): The unique identifier name for the node, defaulting to "GenerateAnswer". |
| 33 | """ |
| 34 | |
| 35 | def __init__( |
| 36 | self, |
| 37 | input: str, |
| 38 | output: List[str], |
| 39 | node_config: Optional[dict] = None, |
| 40 | node_name: str = "SearchLinks", |
| 41 | ): |
| 42 | super().__init__(node_name, "node", input, output, 1, node_config) |
| 43 | |
| 44 | if node_config.get("filter_links", False) or "filter_config" in node_config: |
| 45 | provided_filter_config = node_config.get("filter_config", {}) |
| 46 | self.filter_config = { |
| 47 | **default_filters.filter_dict, |
| 48 | **provided_filter_config, |
| 49 | } |
| 50 | self.filter_links = True |
| 51 | else: |
| 52 | self.filter_config = None |
| 53 | self.filter_links = False |
| 54 | |
| 55 | self.verbose = node_config.get("verbose", False) |
| 56 | self.seen_links = set() |
| 57 | |
| 58 | def _is_same_domain(self, url, domain): |
| 59 | if not self.filter_links or not self.filter_config.get( |
| 60 | "diff_domain_filter", True |
| 61 | ): |
| 62 | return True |
| 63 | parsed_url = urlparse(url) |
| 64 | parsed_domain = urlparse(domain) |
| 65 | return parsed_url.netloc == parsed_domain.netloc |
| 66 | |
| 67 | def _is_image_url(self, url): |
| 68 | if not self.filter_links: |
| 69 | return False |
| 70 | image_extensions = self.filter_config.get("img_exts", []) |
| 71 | return any(url.lower().endswith(ext) for ext in image_extensions) |
| 72 | |
| 73 | def _is_language_url(self, url): |
| 74 | if not self.filter_links: |
| 75 | return False |
no outgoing calls