# Apply to all user agents
User-agent: *
Crawl-delay: 10

Disallow: /preview/
Disallow: /checkout/
Disallow: /user-dashboard/
Disallow: /2025mediakits/

# Allow these bots to crawl the site, except for checkout and dashboard pages
User-agent: Twitterbot
Disallow: /checkout/
Disallow: /user-dashboard/

User-agent: facebookexternalhit
Disallow: /checkout/
Disallow: /user-dashboard/

User-agent: EchoboxBot
Disallow: /checkout/
Disallow: /user-dashboard/

# AI Bots To Block
User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

# Sitemaps
Sitemap: https://www.autonews.com/arc/outboundfeeds/sitemap-index/
Sitemap: https://www.autonews.com/arc/outboundfeeds/sitemap-news-index/
Sitemap: https://www.autonews.com/arc/outboundfeeds/sitemap-section-index/