# The Deep Feed — robots.txt
# Site: https://www.thedeepfeed.ai
# Policy: crawl allowed; search=yes, ai-input=yes, ai-train=no via Content-Signal.
# Agent-readable index: https://www.thedeepfeed.ai/llms.txt
# Full corpus: https://www.thedeepfeed.ai/llms-full.txt

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ora-agent
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: meta-externalagent
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: YouBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Mistral-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Diffbot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ByteSpider
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Omgilibot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /


Sitemap: https://www.thedeepfeed.ai/sitemap-index.xml
Sitemap: https://www.thedeepfeed.ai/sitemap-news.xml
Schemamap: https://www.thedeepfeed.ai/schema-map.xml
Host: www.thedeepfeed.ai