# Standard search engines

User-agent: Googlebot

Allow: /

User-agent: Bingbot

Allow: /

User-agent: YandexBot

Allow: /

User-agent: Applebot

Allow: /

User-agent: DuckDuckBot

Allow: /

# Social previews

User-agent: Twitterbot

Allow: /

User-agent: facebookexternalhit

Allow: /

User-agent: LinkedInBot

Allow: /

# AI and LLM crawlers (training and grounding)

User-agent: GPTBot

Allow: /

User-agent: ChatGPT-User

Allow: /

User-agent: OAI-SearchBot

Allow: /

User-agent: Google-Extended

Allow: /

User-agent: ClaudeBot

Allow: /

User-agent: anthropic-ai

Allow: /

User-agent: PerplexityBot

Allow: /

User-agent: Applebot-Extended

Allow: /

User-agent: CCBot

Allow: /

User-agent: Bytespider

Allow: /

# Catch-all for any unlisted crawler

User-agent: *

Allow: /

Disallow: /blog/car-finance-mis-selling-2026

Disallow: /blog/insurance-job-title-arbitrage

# Sitemap location

Sitemap: https://revn.co.uk/sitemap.xml