User-agent: *
Allow: /

# Disallow admin and auth routes
Disallow: /admin/
Disallow: /login
Disallow: /register
Disallow: /password/
Disallow: /dashboard
Disallow: /settings/
Disallow: /_boost/

# Sitemap locations
Sitemap: https://jugglehire.com/sitemap.xml

# Standard search engine crawl-delay
Crawl-delay: 10

# Google Bot (standard search)
User-agent: Googlebot
Allow: /
Crawl-delay: 2

# Bing Bot (standard search)
User-agent: Bingbot
Allow: /
Crawl-delay: 2

# ===== AI Search Engines & LLMs =====

# OpenAI ChatGPT
User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 5

# Anthropic Claude
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: ClaudeBot
Allow: /
Crawl-delay: 5

# Google AI (Gemini, Bard)
User-agent: Google-Extended
Allow: /
Crawl-delay: 5

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 5

# You.com
User-agent: YouBot
Allow: /
Crawl-delay: 5

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /
Crawl-delay: 5

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Crawl-delay: 5

# Meta AI (Facebook, Instagram)
User-agent: FacebookBot
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 5

# Diffbot (AI web data extraction)
User-agent: Diffbot
Allow: /
Crawl-delay: 5

# Cohere AI
User-agent: cohere-ai
Allow: /
Crawl-delay: 5