# Whispers from the Star - AI-Powered Interactive Storytelling
# SEO-optimized robots.txt for search engines and AI crawlers

# General search engine rules
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /_next/
Disallow: /static/
Disallow: /404
Disallow: /500
Disallow: /*.json$

# Sitemap location
Sitemap: https://whisperfromthestar.org/sitemap.xml

# AI crawler specific rules - Allow access to content
User-agent: GPTBot
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: GoogleOther
User-agent: DuckAssistBot
User-agent: ChatGPT-User
User-agent: GPT-4
User-agent: Claude
User-agent: Bard
User-agent: PaLM
User-agent: Gemini
User-agent: OpenAI-SearchBot
Allow: /
Allow: /index.html
Allow: /about.html
Allow: /contact.html
Allow: /blog.html
Allow: /blog-*.html
Allow: /news.html
Allow: /community.html
Allow: /faq.html
Allow: /technology.html
Allow: /privacy.html
Allow: /terms.html

# Google search engine optimization
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /_next/
Disallow: /static/
Disallow: /404
Disallow: /500
Disallow: /*.json$

# Google AdSense crawler
User-agent: Mediapartners-Google
Allow: /

# Other major search engines
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
Allow: /
Disallow: /admin/
Disallow: /private/

# SEO tool crawlers - Allow access for SEO analysis
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
Allow: /
Disallow: /admin/
Disallow: /private/

# Crawl delay setting
Crawl-delay: 1