| 4 | import cheerio from 'cheerio' |
| 5 | |
| 6 | export default function domwaiter(pages, opts = {}) { |
| 7 | const emitter = new EventEmitter() |
| 8 | |
| 9 | const defaults = { |
| 10 | parseDOM: true, |
| 11 | json: false, |
| 12 | maxConcurrent: 5, |
| 13 | minTime: 500, |
| 14 | } |
| 15 | opts = Object.assign(defaults, opts) |
| 16 | |
| 17 | const limiter = new Bottleneck(opts) |
| 18 | |
| 19 | pages.forEach((page) => { |
| 20 | limiter.schedule(getPage, page, emitter, opts) |
| 21 | }) |
| 22 | |
| 23 | limiter |
| 24 | .on('idle', () => { |
| 25 | emitter.emit('done') |
| 26 | }) |
| 27 | .on('error', (err) => { |
| 28 | emitter.emit('error', err) |
| 29 | }) |
| 30 | |
| 31 | return emitter |
| 32 | } |
| 33 | |
| 34 | async function getPage(page, emitter, opts) { |
| 35 | emitter.emit('beforePageLoad', page) |