# Robots.txt for CITAGEN - AI Visibility Platform
# Optimized for AI search engines and crawlers

User-agent: *
Allow: /

# AI Search Engine Crawlers - Explicitly Allow
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

# Major Search Engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Social Media Crawlers
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

# Product Feeds - Explicitly Allow for AI Shopping
User-agent: GPTBot
Allow: /feeds/
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /feeds/

User-agent: PerplexityBot
Allow: /feeds/
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /feeds/

User-agent: Claude-Web
Allow: /feeds/

User-agent: Google-Extended
Allow: /feeds/

# Disallow admin and sensitive areas
Disallow: /api/
Disallow: /admin/
Disallow: /_redirects

# Allow product feeds
Allow: /feeds/*/products.json

# Important Files for AI and Search Engines

# Sitemaps for traditional search engines
Sitemap: https://citagen.com/sitemap.xml
Sitemap: https://citagen.com/sitemap-products.xml

# AI Discovery Files (Critical for AEO Score 90+)
# LLMS.txt - Structured information for AI language models
# Contains: License, sitemap references, schema info, API endpoints
# Learn more: https://llmstxt.org/
AI-Discovery: https://citagen.com/llms.txt

# AI.txt - AI crawler directives and policies
AI-Policy: https://citagen.com/ai.txt

# Voice.txt - Voice assistant optimization
# For: Alexa, Siri, Google Assistant, ChatGPT Voice, Claude Voice
Voice-Discovery: https://citagen.com/voice.txt

# Crawl Rate Guidelines
Crawl-delay: 5
Request-rate: 100/m

# AI Crawler Specific Preferences
User-agent: GPTBot
Crawl-delay: 1
Max-rate: 100/m
Allow: /methodology
Allow: /pricing
Allow: /faq

User-agent: ClaudeBot
Crawl-delay: 1
Max-rate: 100/m
Allow: /methodology
Allow: /pricing
Allow: /faq

User-agent: PerplexityBot
Crawl-delay: 1
Max-rate: 100/m
Allow: /methodology
Allow: /pricing
Allow: /faq

User-agent: anthropic-ai
Crawl-delay: 1
Allow: /

User-agent: Google-Extended
Allow: /
Crawl-delay: 2

# Last Updated
Last-Modified: 2026-02-02