Files
jiebanke/website/robots.txt

18 lines
268 B
Plaintext

User-agent: *
Allow: /
Sitemap: https://jiebanke.com/sitemap.xml
# 禁止爬虫访问以下路径
Disallow: /merchant/apply.html
Disallow: /contact.html
# 允许的爬虫
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: Baiduspider
Allow: /