# =============================================================
# Content Signals (draft-romm-aipref-contentsignals)
# Declares site-wide preferences for how content may be used.
#   search    = traditional search indexing
#   ai-input  = inference-time use (RAG, agent citations, answering user queries)
#   ai-train  = training / fine-tuning AI models
# =============================================================
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: *
Allow: /

# =============================================================
# AI SEARCH / RETRIEVAL BOTS — ALLOWED
# These bots fetch pages so LLMs can cite this site in answers.
# They do NOT use content for model training.
# =============================================================

# OpenAI ChatGPT Search index
User-agent: OAI-SearchBot
Allow: /

# OpenAI on-demand fetch (when a user asks about or links to a page)
User-agent: ChatGPT-User
Allow: /

# Anthropic on-demand fetch (when Claude needs to cite a page)
User-agent: Claude-User
Allow: /

# Anthropic search index crawler
User-agent: Claude-SearchBot
Allow: /

# Perplexity search index
User-agent: PerplexityBot
Allow: /

# Perplexity on-demand fetch
User-agent: Perplexity-User
Allow: /

# DuckDuckGo AI assistant
User-agent: DuckAssistBot
Allow: /

# You.com search
User-agent: YouBot
Allow: /

# =============================================================
# AI TRAINING BOTS — BLOCKED
# These bots ingest content into model training datasets.
# =============================================================

# OpenAI training crawler
User-agent: GPTBot
Disallow: /

# Google AI training (Gemini, Vertex). Note: Googlebot stays allowed for Search / AI Overviews.
User-agent: Google-Extended
Disallow: /

# Anthropic training crawler
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# ByteDance
User-agent: Bytespider
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# Common Crawl (used by many training pipelines)
User-agent: CCBot
Disallow: /

# Meta
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Apple AI training (Applebot — Apple Search — stays allowed by default)
User-agent: Applebot-Extended
Disallow: /

# Amazon
User-agent: Amazonbot
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# Omgili / Webz.io
User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

# Timpi
User-agent: Timpibot
Disallow: /

# AI2 (Allen Institute)
User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

# img2dataset (LAION)
User-agent: img2dataset
Disallow: /

# iaskspider
User-agent: iaskspider
Disallow: /

# Kangaroo Bot
User-agent: KangarooBot
Disallow: /

# PetalBot (Huawei)
User-agent: PetalBot
Disallow: /

# Sitemap
Sitemap: https://lawsofsoftwareengineering.com/sitemap.xml
Sitemap: https://lawsofsoftwareengineering.com/sitemap-images.xml