# Marselia : https://marselia.app
#
# Everything published here is meant to be read, indexed and quoted, with
# a link back. The full reasoning behind the AI policy below, and the
# sources it was checked against, are in docs/ai-crawlers.md in the
# repository. Short version: crawlers that can send a reader back are
# welcome; crawlers that only build a training corpus are not.
#
# Questions: https://marselia.app/legal/contact

Content-Signal: search=yes, ai-input=yes, ai-train=no

# ── Everyone else ────────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
#
# One worked example of the commit-reveal proof, published on purpose:
# a real draw of ours, taken from /examples. Anyone may read its
# verification page and recompute it, reader or answer engine alike.
# Every other /verify/<id> belongs to the reader who holds the link.
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
#
# The public read-only API that llms.txt offers to agents: the card
# corpus, the knowledge endpoint and the API catalogue. Nothing else
# under /api/ is open.
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# ── Answer engines: welcome ──────────────────────────────────────────
# These build the indexes that AI answers cite, or fetch a page at the
# moment a reader asks a question. Both send readers back.

# OpenAI (search): builds the ChatGPT search index. Refusing it removes us from ChatGPT answers.
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# OpenAI (live): fetches a page when a reader asks. This is the fetch behind the citation.
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Perplexity (search): the Perplexity index. Documented as not used for model training.
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Perplexity (live): reader-initiated fetch, produces the link.
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Anthropic (search): the index behind Claude's web search.
User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Anthropic (live): reader-initiated fetch from Claude.
User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Google (search): grounding in Gemini Apps, which cites and links. No effect on Google Search either way.
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Meta (search): the index Meta AI cites and links from.
User-agent: Meta-WebIndexer
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Meta (live): reader-initiated link fetch.
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Mistral (live): Le Chat's fetch. A French answer engine matters for a French corpus.
User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# DuckDuckGo (live): DuckDuckGo AI answers.
User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Amazon (search): Amazon's answer surfaces.
User-agent: Amzn-SearchBot
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# Apple (search): Siri, Spotlight and Safari search.
User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /verify/
Disallow: /*/verify/
Disallow: /admin
Disallow: /*/admin
Allow: /verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /fr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /es/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /it/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ar/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /tr/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /de/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /pt/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /nl/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /zh/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ja/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /ko/verify/e30f96cc-6843-4801-afd7-7fd6c792cd70
Allow: /api/v1$
Allow: /api/v1/cards
Allow: /api/v1/decks
Allow: /api/v1/knowledge
Allow: /api/v1/academy

# ── Training corpora and resellers: refused ──────────────────────────
# None of these can return a reader. This is also our machine-readable
# reservation of text-and-data-mining rights (art. L. 122-5-3 CPI).

# OpenAI (training): training corpus only. OAI-SearchBot above is the one that cites us.
User-agent: GPTBot
Disallow: /

# Anthropic (training): training corpus only. Claude-SearchBot above is the one that cites us.
User-agent: ClaudeBot
Disallow: /

# Anthropic (legacy) (training): retired token, refused for completeness.
User-agent: anthropic-ai
Disallow: /

# Common Crawl (training): open corpus that feeds third-party training sets. No user-facing product.
User-agent: CCBot
Disallow: /

# Meta (training): training corpus. Meta-WebIndexer above is the one that cites us.
User-agent: Meta-ExternalAgent
Disallow: /

# Apple (training): Apple Intelligence training. Applebot above keeps us in Apple search.
User-agent: Applebot-Extended
Disallow: /

# Amazon (training): documented as possibly training Amazon models.
User-agent: Amazonbot
Disallow: /

# Google (training): crawls on a site owner request we have not made.
User-agent: Google-CloudVertexBot
Disallow: /

# ByteDance (resale): no documentation, no verification method, no audience of ours.
User-agent: Bytespider
Disallow: /

# Diffbot (resale): one crawl resold to several AI firms, with no attribution.
User-agent: Diffbot
Disallow: /

# Timpi (resale): high-volume sweep, no user-facing product.
User-agent: Timpibot
Disallow: /

# Webz.io (resale): crawls to resell a web-data feed.
User-agent: omgili
Disallow: /

# Webz.io (resale): the named form of the omgili crawler, retired by Webz.io and refused like it.
User-agent: Omgilibot
Disallow: /

# Webz.io (resale): successor of omgili: crawls and structures pages for the Webz.io data feed it sells.
User-agent: Webzio
Disallow: /

# Webz.io (training): marks Webz.io data as usable for AI and machine-learning training.
User-agent: Webzio-Extended
Disallow: /

# Allen Institute for AI (training): collects pages to train open language models. No user-facing product.
User-agent: AI2Bot
Disallow: /

# Cohere (training): the token Cohere documents for any future training crawl. Refused in advance.
User-agent: Coherebot
Disallow: /

# Hive (ImageSift) (resale): indexes images for Hive's commercial web-intelligence products. No reader comes back.
User-agent: ImagesiftBot
Disallow: /

# ── Where everything is ──────────────────────────────────────────────
Sitemap: https://marselia.app/sitemap.xml
# Site summary for language models: https://marselia.app/llms.txt