(self)
| 427 | |
| 428 | @threading_helper.reap_threads |
| 429 | def testRead(self): |
| 430 | # Test that reading a weird robots.txt doesn't fail. |
| 431 | addr = self.server.server_address |
| 432 | url = f'http://{socket_helper.HOST}:{addr[1]}' |
| 433 | robots_url = url + '/robots.txt' |
| 434 | parser = urllib.robotparser.RobotFileParser() |
| 435 | parser.set_url(robots_url) |
| 436 | parser.read() |
| 437 | # And it can even interpret the weird paths in some reasonable way. |
| 438 | agent = 'test_robotparser' |
| 439 | self.assertTrue(parser.can_fetch(agent, robots_url)) |
| 440 | self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) |
| 441 | self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) |
| 442 | self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) |
| 443 | self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) |
| 444 | self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) |
| 445 | self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) |
| 446 | self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) |
| 447 | self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) |
| 448 | |
| 449 | |
| 450 | class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): |
nothing calls this directly
no test coverage detected