# LLMS Central - The Robots.txt for AI
# Welcome search engines and AI crawlers! This site helps you find llms.txt files.

User-agent: *
Allow: /

# High-value pages for crawlers
Allow: /domain/
Allow: /category/
Allow: /search
Allow: /api/llms/
Allow: /api/search

# Allow Next.js assets (override query-string disallow below for images)
Allow: /_next/image
Allow: /_next/static/

# Block admin and internal pages
Disallow: /dashboard
Disallow: /api/submit
Disallow: /api/cron/
Disallow: /api/update-domain

# Block duplicate content
Disallow: /*?*
Allow: /search?q=*

# Sitemap location
Sitemap: https://llmscentral.com/sitemap.xml

# Crawl delay for respectful crawling
Crawl-delay: 1

# Special instructions for AI training crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

User-agent: CCBot
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Crawl-delay: 2

User-agent: Claude-Web
Allow: /
Crawl-delay: 2

# This robots.txt is optimized for both search engines and AI crawlers
# Last updated: September 2025