# The Agent2Agent Registry
# Robots.txt for search engine optimization

User-agent: *
Allow: /

# Main pages
Allow: /
Allow: /docs
Allow: /docs/*

# Static assets
Allow: /assets/
Allow: /images/
Allow: /*.css
Allow: /*.js
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.ico
Allow: /*.webp

# Disallow development files
Disallow: /admin/
Disallow: /api/internal/
Disallow: /.env
Disallow: /config/
Disallow: /tmp/

# Sitemap location
Sitemap: https://a2areg.com/sitemap.xml

# Cache friendly for crawlers
Crawl-delay: 1

# Additional crawler directives for better SEO
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# AI/LLM crawlers for better indexing
User-agent: OpenAI-GPTBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Anthropic-AI
Allow: /

# Enterprise search engines
User-agent: MicrosoftPreview
Allow: /

User-agent: AppleBot
Allow: /