visitlex.com

# llms.txt for visitlex.com
# ============================================
# This file provides instructions for Large Language Models (LLMs) and AI agents
# on how they may use content from this website.
#
# Our goal is to ensure that publicly available information about Lexington, KY,
# is accessible and accurate in AI-powered tools, as this helps promote tourism.
# At the same time, we wish to prevent the use of non-public, dynamic, or
# administrative content for training purposes.
#
# Last updated: 2025-06-26

# The sitemap provides a guide to the site's most important content.
Sitemap: https://www.visitlex.com/sitemap.xml


# --- General Rules for All LLM User Agents ---
# By default, we allow the use of our public-facing content for training purposes.
# This includes information on things to do, restaurants, hotels, events, and blog articles.
User-agent: *

# Disallow crawling of internal search result pages. These pages are dynamic
# and do not represent unique, high-quality content for training.
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=

# Disallow any potential administrative, user-specific, or API paths.
# The path "/admin" is the default for Craft CMS.
Disallow: /admin/
Disallow: /login/
Disallow: /account/
Disallow: /api/


# --- Specific Rules for Common Crawl ---
# Common Crawl is a major source of web data for training models.
# We apply the same rules to it to encourage use of our public content.
User-agent: CCBot
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=
Disallow: /admin/


# --- Specific Rules for Google's Extended User Agent ---
# This agent is used for Google's generative AI models (e.g., Bard, Vertex AI).
# We explicitly allow its use on our public content.
User-agent: Google-Extended
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=
Disallow: /admin/


# --- Specific Rules for OpenAI's User Agent ---
# This agent is used by OpenAI models (e.g., ChatGPT).
User-agent: ChatGPT-User
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=
Disallow: /admin/