# robots.txt for Sovereign AI Act
#
# Strategic posture (2026): CITE US, DO NOT INGEST US.
#   • Live AI answer / citation crawlers (ChatGPT search, Claude browsing,
#     Perplexity, Google AI Overviews) — ALLOWED. They drive referral traffic.
#   • AI training crawlers — DISALLOWED. Reserved under the EU Copyright
#     Directive (Directive (EU) 2019/790) Article 4 text-and-data-mining
#     opt-out, machine-expressed via TDMRep (W3C). EU AI Act Article 53(1)(c)
#     requires general-purpose AI model providers to respect this reservation.
#
# Supplemented by: /.well-known/tdmrep.json · tdm-reservation HTTP header
#                  · /ai.txt · /llms.txt · page-level TDM metadata.

# ─── Catch-all + classical search ────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /api/
Disallow: /.well-known/
Allow: /.well-known/tdmrep.json

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: FacebookBot
Allow: /

# ─── AI ANSWER + LIVE CITATION CRAWLERS — ALLOWED (cite + refer traffic) ─
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# ─── AI TRAINING CRAWLERS — DISALLOWED (EU TDM Art. 4 opt-out) ──────────
User-agent: GPTBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: YouBot
Disallow: /

Sitemap: https://www.regulatoryai.eu/sitemap-index.xml
Sitemap: https://www.regulatoryai.eu/sitemap.xml