# ============================================
# robots.txt - 网站爬虫访问控制规则
# 最后更新: 2026-04-20
# ============================================

# ====================
# 1. 通用规则（所有爬虫）
# ====================
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /manager/
Disallow: /runtime/
Disallow: /application/
Disallow: /thinkphp/
Disallow: /extend/
Disallow: /lib/
Disallow: /docs/
Disallow: /check_tables.php
Disallow: /update_sitemap.sh
Disallow: /phpunit.xml
Disallow: /composer.json
Disallow: /.gitignore
Disallow: /.htaccess
Disallow: /.travis.yml

# ====================
# 2. 搜索引擎友好规则
# ====================

## 百度蜘蛛
User-agent: Baiduspider
Allow: /
Crawl-delay: 0

User-agent: Baiduspider-mobile
Allow: /
Crawl-delay: 0

## 谷歌蜘蛛
User-agent: Googlebot
Allow: /
Crawl-delay: 1

## 必应蜘蛛
User-agent: Bingbot
Allow: /
Crawl-delay: 0

## 360蜘蛛
User-agent: 360Spider
Allow: /
Crawl-delay: 1

# ====================
# 3. 禁止的恶意爬虫
# ====================

## 网站审计爬虫
User-agent: meta-externalagent
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: MJ12bot
Disallow: /

## 亚马逊爬虫
User-agent: Amazonbot
Disallow: /

# ====================
# 4. AI大模型爬虫禁止规则
# ====================

## OpenAI/ChatGPT系列
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OpenAI
Disallow: /

## Anthropic/Claude系列
User-agent: ClaudeBot
Disallow: /

User-agent: Anthropic
Disallow: /

## Google AI系列
User-agent: Google-Extended
Disallow: /

## 中文AI模型
User-agent: 文心一言
Disallow: /

User-agent: ERNIE
Disallow: /

User-agent: 通义千问
Disallow: /

User-agent: Qwen
Disallow: /

User-agent: ChatGLM
Disallow: /

User-agent: 讯飞星火
Disallow: /

User-agent: Spark
Disallow: /

User-agent: 豆包
Disallow: /

User-agent: Kimi
Disallow: /

## 通用AI爬虫标识
User-agent: ai-bot
Disallow: /

User-agent: AIBot
Disallow: /

User-agent: ai_crawler
Disallow: /

# ====================
# 5. 通用爬虫限制规则
# ====================

## 所有bot类爬虫（通用限制）
User-agent: *bot*
Crawl-delay: 10
Disallow: /admin/
Disallow: /config/
Disallow: /application/
Disallow: /thinkphp/
Disallow: /extend/
Disallow: /lib/

# ====================
# 6. 网站地图
# ====================
Sitemap: https://www.yuanyeer.com/yy_sitemap_index.xml