# Sidetrain Robots.txt
# https://sidetrain.com

# ============================================
# AI CRAWLERS & LLM BOTS
# ============================================

# OpenAI GPTBot - Used for training and plugins
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.yaml
Allow: /.well-known/ai-plugin.json
Crawl-delay: 2

# OpenAI ChatGPT User Agent
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

# Anthropic Claude
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.yaml
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Crawl-delay: 2

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.yaml
Crawl-delay: 2

# Google Gemini / Bard
User-agent: Google-Extended
Allow: /
Crawl-delay: 2

# Cohere AI
User-agent: cohere-ai
Allow: /
Crawl-delay: 2

# Meta AI
User-agent: FacebookBot
Allow: /
Crawl-delay: 2

User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 2

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /
Crawl-delay: 5

# Apple AI (Applebot-Extended for AI features)
User-agent: Applebot-Extended
Allow: /
Crawl-delay: 2

# ============================================
# SEARCH ENGINE CRAWLERS
# ============================================

User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: *
Allow: /
Crawl-delay: 5

# ============================================
# SITEMAPS & FEEDS
# ============================================

Sitemap: https://sidetrain.com/sitemap.xml

# LLM Context Files (for AI assistants and chatbots)
# LLMs-Txt: https://sidetrain.com/llms.txt
# LLMs-Full: https://sidetrain.com/llms-full.txt
# OpenAPI: https://sidetrain.com/openapi.yaml
# AI-Plugin: https://sidetrain.com/.well-known/ai-plugin.json

# ============================================
# PRIVATE / AUTHENTICATED PAGES
# ============================================

Disallow: /dashboard
Disallow: /dashboard/learner
Disallow: /dashboard/mentor
Disallow: /profile
Disallow: /messages
Disallow: /favorites
Disallow: /admin
Disallow: /session/
Disallow: /session-room/
Disallow: /mentor-setup
Disallow: /mentor-onboarding
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /embed/
Disallow: /calculator/
Disallow: /login
Disallow: /signup

# Block query parameters that create duplicate content
Disallow: /*?payment=
Disallow: /*?redirect=