18 lines
268 B
Plaintext
18 lines
268 B
Plaintext
User-agent: *
|
|
Allow: /
|
|
|
|
Sitemap: https://jiebanke.com/sitemap.xml
|
|
|
|
# 禁止爬虫访问以下路径
|
|
Disallow: /merchant/apply.html
|
|
Disallow: /contact.html
|
|
|
|
# 允许的爬虫
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
|
|
User-agent: Baiduspider
|
|
Allow: / |