# robots.txt — SetKernel Digital Inc.
#
# Content Signals (https://contentsignals.org) — a machine-readable statement of
# how this content may be used. "yes" = permitted, "no" = not permitted.
#   search    : build a search index and show this content in search results
#   ai-input  : use this content as input to an AI model for answers
#               (retrieval-augmented generation, grounding, AI search answers)
#   ai-train  : train or fine-tune AI models on this content

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# AI answer-engine + search crawlers, explicitly welcomed. Cloudflare's managed
# AI-bot blocking is OFF, so these rules are authoritative at the edge. This list
# is the single source in src/config/crawlers.ts (shared with /ai.txt).
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: DuckAssistBot
Allow: /

Sitemap: https://setkernel.com/sitemap.xml