| 30 | main() |
| 31 | |
| 32 | async function main() { |
| 33 | const start = process.hrtime.bigint() |
| 34 | const opts = { |
| 35 | maxConcurrent: MAX_CONCURRENT, |
| 36 | minTime: MIN_TIME, |
| 37 | } |
| 38 | const limiter = new Bottleneck(opts) |
| 39 | const allPages = await loadPages() |
| 40 | const languageCodes = |
| 41 | [languageCode] || |
| 42 | Object.keys(languages) |
| 43 | .filter((language) => language !== 'en') |
| 44 | .map((language) => languages[language].code) |
| 45 | const versions = singleVersion ? [singleVersion] : Object.keys(allVersions) |
| 46 | |
| 47 | const allPermalinks = [] |
| 48 | for (const language of languageCodes) { |
| 49 | for (const version of versions) { |
| 50 | const pages = allPages.filter( |
| 51 | (page) => page.languageCode === language && page.applicableVersions.includes(version) |
| 52 | ) |
| 53 | |
| 54 | const permalinks = pages |
| 55 | .map((page) => { |
| 56 | return page.permalinks.find((permalink) => { |
| 57 | return permalink.languageCode === language && permalink.pageVersion === version |
| 58 | }) |
| 59 | }) |
| 60 | .map((permalink) => { |
| 61 | permalink.url = `http://localhost:${port}${permalink.href}` |
| 62 | return permalink |
| 63 | }) |
| 64 | allPermalinks.push(...permalinks) |
| 65 | } |
| 66 | } |
| 67 | allPermalinks.forEach((page) => { |
| 68 | limiter.schedule(getPage, page) |
| 69 | }) |
| 70 | |
| 71 | limiter |
| 72 | .on('idle', () => { |
| 73 | const end = process.hrtime.bigint() |
| 74 | console.log(`Took ${Number(end - start) / 1000000000}s`) |
| 75 | console.log('All done!') |
| 76 | }) |
| 77 | .on('error', (err) => { |
| 78 | console.log('error', err) |
| 79 | }) |
| 80 | } |
| 81 | |
| 82 | async function getPage(page) { |
| 83 | try { |