# robots.txt - Haveny LLC
# https://haveny.co/robots.txt
# Revised 2026-08-19. Change: citation crawlers unblocked, training crawlers still blocked.
# Revised 2026-09-07. Change: RSL 1.0 licence declared. Terms: use it if you cite it;
#                     training requires a separate written licence.

# ---------------------------------------------------------------
# CONTENT LICENCE (RSL 1.0 - https://rslstandard.org/rsl)
# Global directive: applies to every client that does not match a
# group carrying its own License directive. MUST stay above the first
# "User-agent:" line to remain global.
# ---------------------------------------------------------------
License: https://haveny.co/rsl.xml

# ---------------------------------------------------------------
# SEARCH AND CITATION CRAWLERS - ALLOWED
# These fetch a page to answer a live user question and return a link.
# Blocking them costs referral traffic and gains nothing.
# ---------------------------------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Perplexity - cites sources with links
User-agent: PerplexityBot
Allow: /

# ChatGPT search and live browsing. NOT the same as GPTBot.
# OAI-SearchBot indexes for ChatGPT Search; ChatGPT-User fetches on user request.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Claude live retrieval and citation
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# ---------------------------------------------------------------
# CONTENT SIGNALS - Haveny's own declaration.
# https://contentsignals.org/
#
# As a condition of accessing this website, you agree to abide by the
# following content signals:
#   (a) If a content-signal = yes, you may collect content for the
#       corresponding use.
#   (b) If a content-signal = no, you may not collect content for the
#       corresponding use.
#
#   search:   building a search index and returning links and short excerpts.
#   ai-input: feeding content into AI models for real-time answers, RAG,
#             or grounding.
#   ai-train: training or fine-tuning AI models.
#   use:      what may be retained and reused. "reference" = index, excerpt,
#             and link back. Not summarize-and-reproduce.
#
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
#
# Full machine-readable terms: https://haveny.co/rsl.xml
# ---------------------------------------------------------------
User-agent: *
Content-signal: search=yes, ai-input=yes, ai-train=no, use=reference
Allow: /

# ---------------------------------------------------------------
# TRAINING-ONLY CRAWLERS - BLOCKED
# These harvest content for model training with no attribution and no traffic back.
# See https://haveny.co/ai.txt for the content usage policy.
# ---------------------------------------------------------------

# OpenAI model training
User-agent: GPTBot
Disallow: /

# Common Crawl - feeds many training corpora
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# ---------------------------------------------------------------
# Google-Extended - ALLOWED as of 2026-09-07. DECIDED, not open.
# It is a single token gating BOTH Gemini training AND Gemini/Vertex
# grounding. Google does not offer the two separately. Allowing it buys
# Gemini citations; the cost is training use, which we do not consent to
# and have reserved against in three places: the Content-Signal above,
# /rsl.xml (payment type="training" - licence required), and /ai.txt.
# It does NOT affect Google Search or AI Overviews either way; those run
# off Googlebot, allowed above.
# ---------------------------------------------------------------
User-agent: Google-Extended
Allow: /

# ---------------------------------------------------------------
# Sitemaps
# ---------------------------------------------------------------
Sitemap: https://haveny.co/sitemap.xml