# robots.txt for aiforcode.io
# Optimized for SEO and GEO (Generative Engine Optimization)

# ===========================================
# Default: Allow all crawlers
# ===========================================
User-agent: *
Allow: /

# Block non-public directories
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.github/
Disallow: /netlify/
Disallow: /data/
Disallow: /tools-data/
Disallow: /icons/

# Block build artifacts and templates
Disallow: /build.js
Disallow: /template.html
Disallow: /tool-template.html
Disallow: /tool-language-template.html
Disallow: /comparison-template.html
Disallow: /comparisonPairs.json
Disallow: /comparisonOverrides.json
Disallow: /tools.json
Disallow: /package.json
Disallow: /package-lock.json
Disallow: /serve.sh
Disallow: /*.json$

# ===========================================
# Search Engine Crawlers (SEO)
# ===========================================
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# ===========================================
# AI/LLM Crawlers (GEO - Generative Engine Optimization)
# Allowing these improves visibility in AI-powered search
# ===========================================
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: YouBot
Allow: /

# ===========================================
# Blocked: Aggressive crawlers with zero referral value
# ===========================================
User-agent: Amazonbot
Disallow: /

# ===========================================
# Sitemap
# ===========================================
Sitemap: https://aiforcode.io/sitemap.xml