# robots.txt for oxone.ca
# Policy: Allow AI search/retrieval crawlers (they can cite us in answers).
# Block AI training crawlers (no dataset harvesting).
#
# 2026-09-06: this file is now the ONLY policy on this zone. Cloudflare's managed
# robots.txt (is_robots_txt_managed) used to PREPEND its own block for ClaudeBot,
# GPTBot, CCBot and Google-Extended on top of everything below, and the zone also
# had ai_bots_protection="block", which 403'd every AI crawler at the edge —
# including OAI-SearchBot and PerplexityBot, which this file explicitly allows.
# So the whole "make our pages quotable in AI answers" plan was being served a 403.
# Both settings are now off; the Content-Signal line CF used to add is declared here.

# -------- Default (search engines + most bots) --------
User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

# Block demo/sample pages from indexing
Disallow: /demo/
Disallow: /sample-realtor
Disallow: /sample-realtor.html
Disallow: /sample-trades
Disallow: /sample-trades.html
Disallow: /sample-restaurant
Disallow: /sample-restaurant.html
Disallow: /sample-clinic
Disallow: /sample-clinic.html
Disallow: /sample-accountant
Disallow: /sample-accountant.html

# -------- AI search / retrieval crawlers (ALLOW — for citations in AI answers) --------
# ChatGPT web search
User-agent: OAI-SearchBot
Allow: /

# Perplexity (citations)
User-agent: PerplexityBot
Allow: /

# Anthropic's user-facing Claude browsing
User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

# ChatGPT live-browse user agent
User-agent: ChatGPT-User
Allow: /

# Anthropic live-browse
User-agent: claude-user
Allow: /

# Google AI overviews (Googlebot already allowed above; Google-Extended controls AI training)
User-agent: Google-Extended
Disallow: /

# Apple AI / Siri
User-agent: Applebot
Allow: /

# Anthropic's general crawler. Allowed 2026-09-06 (Chen): Claude reaches a site
# through ClaudeBot, so blocking it means Claude cannot cite us at all.
User-agent: ClaudeBot
Allow: /

# Common Crawl. Allowed 2026-09-06 (Chen): a large share of retrieval-based systems
# read the Common Crawl corpus, and being absent from it is a permanent absence.
User-agent: CCBot
Allow: /

# -------- AI training crawlers (BLOCK — no dataset harvesting) --------
User-agent: GPTBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Diffbot
Disallow: /

# Sitemap location
Sitemap: https://oxone.ca/sitemap.xml