# robots.txt for https://www.codebuddy.cn/
# Last updated: 2026-07-24
# Maintainer: CodeBuddy SEO Team
# Spec: https://www.rfc-editor.org/rfc/rfc9309 (Robots Exclusion Protocol)

# ─────────────────────────────────────────────────────────────
# 通用规则：允许所有合规搜索引擎抓取主要内容
# ─────────────────────────────────────────────────────────────
User-agent: *
Allow: /
# 屏蔽认证/账户/工作台等用户私域路径
Disallow: /login
Disallow: /login/
Disallow: /register
Disallow: /register/
Disallow: /logout
Disallow: /usercenter
Disallow: /usercenter/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /settings
Disallow: /settings/
Disallow: /profile
Disallow: /profile/
Disallow: /admin
Disallow: /admin/
Disallow: /no-permission
Disallow: /no-permission/
Disallow: /redirect
Disallow: /redirect/
Disallow: /setup
Disallow: /setup/
Disallow: /invite
Disallow: /invite/
# 屏蔽内部接口
Disallow: /api/
Disallow: /trpc/
Disallow: /_next/
Disallow: /assets/
Disallow: /fonts/
# 活动/裂变/中转类页面（有短时效或参数化，不适合被通用检索收录）
Disallow: /fission
Disallow: /fission/
Disallow: /promotion
Disallow: /promotion/
Disallow: /quota
Disallow: /quota/
Disallow: /app-download
Disallow: /app-download/
Disallow: /workbuddy-mp
Disallow: /workbuddy-mp/
# 屏蔽常见跟踪参数（带参 URL 容易产生重复内容）
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*?ref=*
Disallow: /*&utm_*
# 抓取节流（避免高峰期占用带宽）
Crawl-delay: 1

# ─────────────────────────────────────────────────────────────
# Google / Bing：放开抓取节流（这两家忽略 Crawl-delay，仅作显式声明）
# ─────────────────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /login
Disallow: /register
Disallow: /usercenter
Disallow: /api/

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /
Disallow: /login
Disallow: /register
Disallow: /usercenter
Disallow: /api/

# ─────────────────────────────────────────────────────────────
# 国内搜索引擎
# ─────────────────────────────────────────────────────────────
User-agent: Baiduspider
Allow: /
Disallow: /login
Disallow: /register
Disallow: /usercenter
Disallow: /api/

User-agent: Sogou web spider
Allow: /
Disallow: /login
Disallow: /register
Disallow: /api/

User-agent: 360Spider
Allow: /
Disallow: /login
Disallow: /api/

User-agent: YisouSpider
Allow: /
Disallow: /api/

# ─────────────────────────────────────────────────────────────
# AI / LLM 抓取代理：允许收录公开内容，提升 GEO（Generative Engine Optimization）可见性
# 注：如希望禁止某家，将对应 User-agent 块下改为 "Disallow: /"
# ─────────────────────────────────────────────────────────────
User-agent: GPTBot                # OpenAI ChatGPT 训练
Allow: /

User-agent: OAI-SearchBot         # OpenAI SearchGPT 检索
Allow: /

User-agent: ChatGPT-User          # OpenAI ChatGPT 实时浏览
Allow: /

User-agent: Google-Extended       # Google Gemini 训练
Allow: /

User-agent: GoogleOther           # Google 多用途研发抓取
Allow: /

User-agent: ClaudeBot             # Anthropic Claude 训练
Allow: /

User-agent: Claude-Web            # Anthropic Claude.ai 浏览
Allow: /

User-agent: anthropic-ai          # Anthropic 通用
Allow: /

User-agent: PerplexityBot         # Perplexity 索引
Allow: /

User-agent: Perplexity-User       # Perplexity 实时浏览
Allow: /

User-agent: cohere-ai             # Cohere
Allow: /

User-agent: Applebot              # Apple Spotlight / Siri
Allow: /

User-agent: Applebot-Extended     # Apple Intelligence 训练
Allow: /

User-agent: Bytespider            # 字节跳动豆包
Allow: /

User-agent: Amazonbot             # Amazon Alexa+ / 训练
Allow: /

User-agent: Meta-ExternalAgent    # Meta AI
Allow: /

User-agent: DuckAssistBot         # DuckDuckGo AI
Allow: /

User-agent: YouBot                # You.com
Allow: /

User-agent: Diffbot               # 知识图谱
Allow: /

User-agent: DoubaoBot             # 豆包
Allow: /

User-agent: YuanbaoBot            # 元宝
Allow: /

User-agent: DeepSeekBot           # DeepSeek
Allow: /

User-agent: Kimibot               # Kimi
Allow: /

User-agent: MoonshotBot           # Moonshot
Allow: /

User-agent: TongyiBot             # Tongyi
Allow: /

User-agent: AliyunBot             # AliyunBot
Allow: /


# ─────────────────────────────────────────────────────────────
# 不希望抓取的低质量/SEO 工具型爬虫
# ─────────────────────────────────────────────────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ─────────────────────────────────────────────────────────────
# Sitemaps（必须用绝对 URL）
# 顶层 sitemap.xml 是 sitemapindex，聚合了 apiDocs 与 docs 两个子域的 sitemap
# ─────────────────────────────────────────────────────────────
Sitemap: https://www.codebuddy.cn/sitemap.xml

# Host 提示（部分搜索引擎使用）
Host: https://www.codebuddy.cn