# Default: allow all
User-agent: *
Allow: /
# Cloudflare Content Signals Policy — https://blog.cloudflare.com/content-signals-policy/
# Declares how well-behaved crawlers may process our content.
#   search    — appearing in traditional search results
#   ai-input  — used as input to AI answers / summaries (cite-and-quote)
#   ai-train  — retained for training AI models
# Matches the "explicitly allowed" AI-crawler blocks below; flip ai-train=no
# if the brand later decides to opt out of training-set inclusion.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Sitemap: https://inspect-solutions.com/sitemap.xml

# ============================================================
# AI / LLM crawlers — explicitly allowed
# See https://llmstxt.org and per-vendor crawler docs.
# A machine-readable site summary lives at /llms.txt
# ============================================================

# OpenAI — ChatGPT
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: ChatGPT-User
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: OAI-SearchBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: Claude-Web
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: anthropic-ai
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Google — Bard / Gemini / AI Overviews
User-agent: Google-Extended
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Perplexity
User-agent: PerplexityBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: Perplexity-User
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Amazon — Alexa / Q
User-agent: Amazonbot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Apple — Apple Intelligence
User-agent: Applebot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: Applebot-Extended
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Meta — Llama / Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: Meta-ExternalFetcher
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: FacebookBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Microsoft — Copilot / Bing Chat
User-agent: bingbot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# DuckDuckGo — DuckAssist
User-agent: DuckAssistBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# You.com
User-agent: YouBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Cohere
User-agent: cohere-ai
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: cohere-training-data-crawler
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Mistral
User-agent: MistralAI-User
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ByteDance (Doubao / Toutiao)
User-agent: Bytespider
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Common Crawl (used by many open AI datasets)
User-agent: CCBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Diffbot
User-agent: Diffbot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Yandex
User-agent: YandexBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes