# seo-tool crawlers: zero search traffic, heavy load (blocked at nginx level too)
User-Agent: AhrefsBot
User-Agent: SemrushBot
User-Agent: SiteAuditBot
User-Agent: MJ12bot
User-Agent: DotBot
User-Agent: BLEXBot
User-Agent: DataForSeoBot
User-Agent: serpstatbot
User-Agent: SEOkicks
User-Agent: Barkrowler
User-Agent: ZoominfoBot
User-Agent: MegaIndex
Disallow: /

# ai crawlers: welcome, but paced - they fetch two thirds of all page views and the listing pages
# they favour are the expensive ones. nginx enforces the same pace on /jobs listings with a 429.
User-Agent: ClaudeBot
User-Agent: GPTBot
User-Agent: OAI-SearchBot
User-Agent: ChatGPT-User
User-Agent: PerplexityBot
User-Agent: Amazonbot
User-Agent: meta-externalagent
User-Agent: Bytespider
User-Agent: CCBot
User-Agent: cohere-ai
User-Agent: Applebot-Extended
User-Agent: Google-Extended
Crawl-delay: 1
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /auth/
Disallow: /billing
Disallow: /login
Disallow: /logout
Disallow: /messages
Disallow: /payment
Disallow: /profile
Disallow: /screening/
Disallow: /settings
Disallow: /signup
Disallow: /verification
Disallow: /jobs?
Disallow: /talents?
Disallow: /trap-

# auth surface: no content to index, and /signup?redirect=<slug> multiplies into one
# url per vacancy - 292k bot fetches a week, a fifth of every bot page view, for nothing.
User-Agent: *
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /auth/
Disallow: /billing
Disallow: /login
Disallow: /logout
Disallow: /messages
Disallow: /payment
Disallow: /profile
Disallow: /screening/
Disallow: /settings
Disallow: /signup
Disallow: /verification
Disallow: /jobs?
Disallow: /talents?
Disallow: /trap-

Sitemap: https://alion.io/sitemap_index.xml