()
| 68 | await q.put(new_url) |
| 69 | |
| 70 | async def worker(): |
| 71 | async for url in q: |
| 72 | if url is None: |
| 73 | return |
| 74 | try: |
| 75 | await fetch_url(url) |
| 76 | except Exception as e: |
| 77 | print(f"Exception: {e} {url}") |
| 78 | dead.add(url) |
| 79 | finally: |
| 80 | q.task_done() |
| 81 | |
| 82 | await q.put(base_url) |
| 83 |