# robots.txt 2019-04-12

# Sensis.com.au
User-agent: Sensis.com.au Web Crawler 
Disallow: /update/
Disallow: /404.html
Disallow: /job_template.html

# Sensis
User-agent: Sensis Web Crawler 
Disallow: /update/
Disallow: /404.html
Disallow: /job_template.html

# Yahoo!
User-agent: Slurp
Crawl-delay: 20
Disallow: /update/
Disallow: /404.html
Disallow: /job_template.html

# Google will not spider
User-agent: Googlebot
Disallow: /update/
Disallow: /404.html
Disallow: /job_template.html

# Bing
User-agent: msnbot/2.0b
Crawl-delay: 100
Disallow: /update/
Disallow: /404.html
Disallow: /job_template.html

# Jooblebot
User-agent: Jooblebot
Disallow: /

# Yandex
User-agent: Yandex
Disallow: /

# MJ12bot
User-agent: MJ12bot
Disallow: /

# AhrefsBot
User-agent: AhrefsBot
Disallow: /

# trovitBot
User-agent: trovitBot
Disallow: /

# sitebot
User-agent: sitebot
Disallow: /

# Others
User-agent: *
Disallow: /update/
Disallow: /404.html
Disallow: /job_template.html