# robot exclusion for site
User-agent: *
Disallow: /dev
Disallow: /c/*/dev
Crawl-delay: 30
#Baiduspider
User-agent: Baiduspider
Disallow: /
User-agent: MJ12bot
Disallow: /