# === PRIMARY AI ASSISTANTS & TRAINING (allow - essential for visibility) ===
# Claude/Anthropic
User-agent: ClaudeBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /

# OpenAI - Training & Search
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# Google - Gemini & Vertex AI
User-agent: Google-Extended
Allow: /
User-agent: Gemini-Deep-Research
Allow: /
User-agent: Google-CloudVertexBot
Allow: /

# Grok (xAI)
User-agent: GrokBot
Allow: /

# === AI SEARCH ENGINES (allow - great for organic reach) ===
# You.com
User-agent: YouBot
Allow: /

# Phind
User-agent: PhindBot
Allow: /

# Exa AI
User-agent: ExaBot
Allow: /

# Andi
User-agent: AndiBot
Allow: /

# DuckDuckGo DuckAssist
User-agent: DuckAssistBot
Allow: /

# === AI MODEL TRAINING PLATFORMS ===
# Common Crawl
User-agent: CCBot
Allow: /

# Amazon/Bedrock
User-agent: bedrockbot
Allow: /

# Mistral AI
User-agent: MistralAI-User
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /
User-agent: cohere-training-data-crawler
Allow: /

# AI2 (Allen Institute)
User-agent: AI2Bot
Allow: /
User-agent: Ai2Bot-Dolma
Allow: /

# Meta/Facebook AI
User-agent: meta-externalagent
Allow: /
User-agent: meta-externalfetcher
Allow: /

# === WEB CRAWLING & CONTENT EXTRACTION ===
User-agent: FirecrawlAgent
Allow: /
User-agent: Diffbot
Allow: /

# === APPLE INTELLIGENCE ===
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# === SPECIALIZED AI TOOLS ===
# ByteDance (TikTok/Douyin)
User-agent: Bytespider
Allow: /

# Image search & analysis
User-agent: ImagesiftBot
Allow: /

# Research crawlers
User-agent: omgili
Allow: /
User-agent: omgilibot
Allow: /
User-agent: webzio-extended
Allow: /

# Data extraction for AI
User-agent: img2dataset
Allow: /

# === Search engines ===
User-agent: Googlebot
Allow: /
User-agent: Googlebot-Image
Allow: /
User-agent: Googlebot-Mobile
Allow: /
User-agent: Googlebot-News
Allow: /
User-agent: Googlebot-Video
Allow: /
User-agent: Mediapartners-Google
Allow: /
User-agent: AdsBot-Google
Allow: /
User-agent: bingbot
Allow: /
User-agent: BingPreview
Allow: /
User-agent: msnbot
Allow: /
User-agent: msnbot-media
Allow: /
User-agent: adidxbot
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Yahoo! Slurp
Allow: /

# === Social media preview bots ===
User-agent: Twitterbot
Allow: /
User-agent: facebookexternalhit
Allow: /
User-agent: LinkedInBot
Allow: /
User-agent: Slackbot
Allow: /
User-agent: WhatsApp
Allow: /
User-agent: TelegramBot
Allow: /
User-agent: redditbot
Allow: /
User-agent: Pinterest
Allow: /

# === Archive ===
User-agent: archive.org_bot
Allow: /

# === SEO tools - throttled or blocked ===
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-BA
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: rogerbot
Disallow: /
User-agent: PiplBot
Disallow: /

# === Regional search engines (block if no traffic from there) ===
User-agent: yandex
Disallow: /
User-agent: Baiduspider
Disallow: /

# === Default: block everything else ===
User-agent: *
Disallow: