User-Agent: *
Disallow: /error
Disallow: /shmsh
Disallow: /index.htm
Disallow: /MicroBlog/m
Disallow: /microblog/m
Crawl-delay: 120
User-Agent: youdao
Crawl-delay: 1200
User-Agent: wukongbot
Disallow: /
User-agent: dotbot 
Disallow: / 
User-agent: ScoutJet
Disallow: /
Allow: /public
# (每5秒不得浏览多于一个页面)
Crawl-delay: 5
User-agent: twiceler
Disallow: /