# Crackle PR — robots.txt
# Default: allow everyone, point at the sitemap index.

User-agent: Googlebot
Allow: /
Disallow: /strategy/

User-agent: Bingbot
Allow: /
Disallow: /strategy/

User-agent: *
Allow: /
Disallow: /strategy/


# ---------------------------------------------------------------------------
# Machine-readable dataset discovery (for dataset harvesters and researchers)
#   /data.json          DCAT / Project Open Data catalog
#   /croissant.jsonld   MLCommons Croissant 1.0 dataset metadata
#   /datapackage.json   Frictionless Data Package (column-level schema)
#   /CITATION.cff       Citation File Format
#   /api/v1/openapi.json  OpenAPI 3.1 spec for the public data API
#   /feeds/ai-visibility-index.xml  New-edition RSS feed
# All datasets CC BY 4.0 — attribute to Crackle PR (https://www.cracklepr.com).
# ---------------------------------------------------------------------------

# ---------------------------------------------------------------------------
# Explicit allows for AI / LLM crawlers (signals consent for training + RAG).
# Removing the block (vs. silence) avoids accidental exclusion by intermediaries
# such as Cloudflare's "Block AI Bots" toggle interpreting silence as opt-out.
# ---------------------------------------------------------------------------

# OpenAI — ChatGPT search + GPT model training
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude (crawl, search grounding, and user-initiated fetch)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /


# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI Overviews / Gemini grounding
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Common Crawl (training corpus for many LLMs)
User-agent: CCBot
Allow: /

# Bytespider (Doubao / TikTok)
User-agent: Bytespider
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# DuckDuckGo Assist / Cohere / You.com
User-agent: DuckAssistBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

# Diffbot — used by several enterprise LLM stacks
User-agent: Diffbot
Allow: /

# Mistral (France)
User-agent: MistralAI-User
Allow: /

# ---------------------------------------------------------------------------
# International search + AI crawlers
# B2B tech buyers at multinationals increasingly query non-US assistants.
# Explicit allow-list so Cloudflare / edge filters don't silently drop them.
# ---------------------------------------------------------------------------

# Baidu (China — largest non-Google search)
User-agent: Baiduspider
Allow: /

# Yandex (Russia / CIS — also powers Yandex GPT / Alice)
User-agent: YandexBot
Allow: /

User-agent: YandexRenderResourcesBot
Allow: /

# Naver (South Korea)
User-agent: Yeti
Allow: /

# Seznam (Czech Republic)
User-agent: SeznamBot
Allow: /

# Sogou (China)
User-agent: Sogou web spider
Allow: /

# Huawei Petal Search (global, esp. APAC/EMEA)
User-agent: PetalBot
Allow: /

# Qihoo 360 (China)
User-agent: 360Spider
Allow: /

# Alibaba Qwen / Tongyi (China)
User-agent: Qwen
Allow: /

# DeepSeek (China)
User-agent: DeepSeekBot
Allow: /

# Moonshot Kimi (China)
User-agent: KimiBot
Allow: /

# Zhipu ChatGLM (China)
User-agent: ChatGLM-Spider
Allow: /

# Kagi (US indie search / assistant)
User-agent: KagiBot
Allow: /

# Neeva-successor / Andi / Phind style research bots
User-agent: PhindBot
Allow: /

# Timpi (decentralized index used by some EU assistants)
User-agent: TimpiBot
Allow: /

Sitemap: https://www.cracklepr.com/sitemap.xml