| 36 | |
| 37 | @asyncio.coroutine |
| 38 | def addurls(self, urls): |
| 39 | for url, parenturl in urls: |
| 40 | url = urllib.parse.urljoin(parenturl, url) |
| 41 | url, frag = urllib.parse.urldefrag(url) |
| 42 | if (url.startswith(self.rooturl) and |
| 43 | url not in self.busy and |
| 44 | url not in self.done and |
| 45 | url not in self.todo): |
| 46 | self.todo.add(url) |
| 47 | yield from self.sem.acquire() |
| 48 | task = asyncio.Task(self.process(url)) |
| 49 | task.add_done_callback(lambda t: self.sem.release()) |
| 50 | task.add_done_callback(self.tasks.remove) |
| 51 | self.tasks.add(task) |
| 52 | |
| 53 | @asyncio.coroutine |
| 54 | def process(self, url): |