# unicrew.com
# AI crawler posture ratified 2026-08-13 (docs/geo-standards.md). All allowed.

# Retrieval crawlers: fetch a page to answer a live question, and cite it.
# These are the ones that return a citation, and the ones the previous file
# left unnamed.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Applebot
Allow: /
Disallow: /cdn-cgi/l/email-protection

# Training crawlers: no citation, but presence in a model's own knowledge is
# worth more to a consultancy than withholding it. Google-Extended affects
# Gemini grounding only and has no effect on Search ranking, so allowing it
# costs nothing in SEO.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
Allow: /
Disallow: /cdn-cgi/l/email-protection

# Everything else that offers no citation return, allowed by the same decision
# rather than by omission.
User-agent: Amazonbot
User-agent: Bytespider
User-agent: meta-externalagent
Allow: /
Disallow: /cdn-cgi/l/email-protection

User-agent: *
Allow: /
Disallow: /cdn-cgi/l/email-protection

Sitemap: https://unicrew.com/sitemap_index.xml