# 5280 Legends - robots.txt
# https://5280locallegends.com
# Optimized for traditional search engines and AI/LLM crawlers

# ===========================================
# SITEMAPS & LLM CONTEXT FILES
# ===========================================

# Primary Sitemap (Static)
Sitemap: https://5280locallegends.com/sitemap.xml

# Dynamic Sitemap (Edge Function - Real-time data)
Sitemap: https://whwhjnjugwlxcmdgnsyu.supabase.co/functions/v1/generate-sitemap

# LLM Context Files
# Static: https://5280locallegends.com/llms.txt
# Static: https://5280locallegends.com/.well-known/llms.txt
# Dynamic (recommended): https://whwhjnjugwlxcmdgnsyu.supabase.co/functions/v1/generate-llms-txt
# Dynamic (full): https://whwhjnjugwlxcmdgnsyu.supabase.co/functions/v1/generate-llms-txt?full=true

# ===========================================
# AI & LLM CRAWLERS (Priority Access)
# ===========================================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# OpenAI ChatGPT User Agent
User-agent: ChatGPT-User
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# Google AI (Gemini, Bard)
User-agent: Google-Extended
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

User-agent: Claude-Web
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# Meta AI
User-agent: FacebookBot
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

User-agent: meta-externalagent
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# Microsoft Copilot / Bing Chat
User-agent: Copilot
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# Common Crawl (used for training data)
User-agent: CCBot
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# You.com AI
User-agent: YouBot
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# Neeva AI (acquired by Snowflake)
User-agent: NeevaBot
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# ===========================================
# TRADITIONAL SEARCH ENGINES
# ===========================================

# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yahoo
User-agent: Slurp
Allow: /
Crawl-delay: 2

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yandex
User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Baidu
User-agent: Baiduspider
Allow: /
Crawl-delay: 3

# ===========================================
# SOCIAL MEDIA CRAWLERS
# ===========================================

# Facebook
User-agent: facebookexternalhit
Allow: /

# Twitter
User-agent: Twitterbot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Pinterest
User-agent: Pinterest
Allow: /

# ===========================================
# APPLE / SIRI
# ===========================================

# Apple (Siri, Spotlight)
User-agent: Applebot
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/

# ===========================================
# DEFAULT RULES
# ===========================================

# Default rule for all other bots
User-agent: *
Allow: /
Allow: /functions/v1/generate-llms-txt
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5