# canlah.ai robots.txt — explicit AI-crawler policy
# Policy: we WANT to be read, cited and recommended by AI engines.
# GEO decision 2026-07: visibility > training-data protection.

# --- AI answer/search crawlers (citations) — explicitly allowed ---
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: GoogleOther
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

# --- AI training crawlers — allowed (they feed future answer corpora) ---
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

# --- Classic search engines ---
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

# --- Everyone else ---
User-agent: *
Allow: /
# /api/ 是 SSR 接口(审计 SSE/报告代理),对引用零价值,爬到只会烧后端配额。
# 报告端点另有 grant-cookie 鉴权,此处是纵深防御。
Disallow: /api/
# 白皮书 PDF 下载路径:留资对价,不给爬虫枚举(spec 2026-08-16)。
# ⚠️ robots.txt 是"最具体组胜出"——具名爬虫组不重复这条就完全吃不到这个门,
#    所以上面每一个具名组都必须带同一行,改动时两边一起改。
Disallow: /whitepaper/dl/
Disallow: /playbook/dl/

Sitemap: https://canlah.ai/sitemap.xml
# Skills Radar 是挂在 /marketing-skills 的独立 Next.js app(仓 Canlah-AI/marketing-skills-hub)。
# 2026-09-03 解锁:该仓已加 app/sitemap.ts 并部署,实测 200 + application/xml,故放开这行。
# ⚠️ 这份 sitemap 只列 44 个 URL(hub/catalog/guide/vision + 10 个岗位页 + 30 个带实测
#    工具 schema 的 skill),不是全部 434 个详情页 —— 其余约 400 页正文是第三方描述套
#    同一模板,与 glama/mcpmarket 上的同名条目近重复,全量索引正是 Google scaled
#    content abuse 点名的形状。它们在页面层是 noindex,follow(可爬、传权重、不参与
#    索引竞争)。要扩批次得先让那些页面有自己的内容,不是把数字调大。
Sitemap: https://canlah.ai/marketing-skills/sitemap.xml