# Generated by scripts/generate-robots.ts from src/data/searchCrawlers.ts. Do not edit by hand. User-agent: * Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request # Advertised open data resources (each states its licence; see /terms#data-licence) Allow: /coverage.json Allow: /pension-portals.json Allow: /scoreboard.csv # --- Search, answer and retrieval crawlers: explicitly allowed --- # Each token is read from its vendor's own documentation (see # src/data/searchCrawlers.ts for the page and the read date). A crawler obeys # only the most specific group that names it, so each named group repeats # the private paths from the * group above. # Brave documents no token of its own and follows the Googlebot group. User-agent: ExaSearchBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: ExaUser Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: PerplexityBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Perplexity-User Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: OAI-SearchBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: ChatGPT-User Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: GPTBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Claude-SearchBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Claude-User Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: ClaudeBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Googlebot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Google-Extended Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: GoogleOther Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Google-CloudVertexBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Applebot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Applebot-Extended Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Bingbot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: DuckAssistBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Meta-ExternalAgent Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Meta-ExternalFetcher Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Meta-WebIndexer Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: FacebookBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: facebookexternalhit Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: MistralAI-User Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: MistralAI-Index Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: cohere-ai Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: YouBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Amazonbot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Amzn-SearchBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Amzn-User Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Claude-Web Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: anthropic-ai Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: CCBot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request User-agent: Twitterbot Allow: / Allow: /api$ Disallow: /admin Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /success Disallow: /mandate Disallow: /login Disallow: /unsubscribe Disallow: /watch/ Disallow: /data-request # --- Bulk scrapers and SEO harvesters that never cite: blocked --- # Search, answer and AI crawlers are allowed above. User-agent: Bytespider Disallow: / User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Diffbot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: Timpibot Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: omgili Disallow: / User-agent: Omgilibot Disallow: / User-agent: magpie-crawler Disallow: / Sitemap: https://pensionhunter.ai/sitemap.xml