| 1 | import fs from 'fs/promises'; |
| 2 | |
| 3 | function createRobotsTxt(domain: string) { |
| 4 | return ` |
| 5 | User-agent: * |
| 6 | Allow: / |
| 7 | |
| 8 | # metrics |
| 9 | Disallow: /ph |
| 10 | Disallow: /pp |
| 11 | |
| 12 | # user only pages |
| 13 | Disallow: /api |
| 14 | Disallow: /all |
| 15 | Disallow: /branding |
| 16 | Disallow: /configurations |
| 17 | Disallow: /inbox |
| 18 | Disallow: /members |
| 19 | Disallow: /metrics |
| 20 | Disallow: /plans |
| 21 | Disallow: /starred |
| 22 | |
| 23 | # Block files ending in .json, _buildManifest.js, _middlewareManifest.js, _ssgManifest.js, and any other JS files |
| 24 | Disallow: /*.json$ |
| 25 | Disallow: /*_buildManifest.js$ |
| 26 | Disallow: /*_middlewareManifest.js$ |
| 27 | Disallow: /*_ssgManifest.js$ |
| 28 | Disallow: /*.js$ |
| 29 | |
| 30 | Sitemap: https://${domain}/sitemap.xml |
| 31 | `.trim(); |
| 32 | } |
| 33 | |
| 34 | export async function buildRobots( |
| 35 | redirectDomain: string | null, |