| 23 | |
| 24 | |
| 25 | def test_robots_node_scrapable(robots_node): |
| 26 | state = {"url": "https://perinim.github.io/robots.txt"} |
| 27 | |
| 28 | # Mocking AsyncChromiumLoader to return a fake robots.txt content |
| 29 | robots_node.AsyncChromiumLoader = MagicMock( |
| 30 | return_value=MagicMock(load=MagicMock(return_value="User-agent: *\nAllow: /")) |
| 31 | ) |
| 32 | |
| 33 | # Execute the node |
| 34 | result_state, result = robots_node.execute(state) |
| 35 | |
| 36 | # Check the updated state |
| 37 | assert result_state["is_scrapable"] == "yes" |
| 38 | assert result == ("is_scrapable", "yes") |
| 39 | |
| 40 | |
| 41 | def test_robots_node_not_scrapable(robots_node): |