User-Agent: OmniExplorer_Bot
Disallow: /

User-agent: *
Crawl-delay: 5
Request-rate: 1/5
Disallow: /admin
Disallow: /all
Disallow: /all/javascript/ 
Disallow: /CVS
Disallow: /imagen_t1msn
Disallow: /lang
Disallow: /SYST
Disallow: /USER/CVS
Disallow: /USER/pricelist
Disallow: /helper
Disallow: /USER/purchase.php
Disallow: /USER/RECR/dyn_frame.php

#cms cleanup
Disallow: /cms/resources/hispanic_recruiter_review*
Disallow: /cms/resources/articles/rating
Disallow: /cms/resources/articles/popular
Disallow: /cms/resources/articles/category
Disallow: /cms/resources/bookreview
Disallow: /cms/resources/breakroom
Disallow: /cms/resources/event_type/conference
Disallow: /cms/resources/expert/
Disallow: /cms/resources/newsletter_type/
Disallow: /cms/resources/publish_area/recruiting_resources
Disallow: /cms/resources/taxonomy/
Disallow: /cms/recruiter/recruiting/category/
Disallow: /cms/helper/

#wildcard matching to disable some junk. reference:http://tools.seobook.com/robots-txt/

Disallow: /*?user
Disallow: /cms/en/careeradvice/event/2007/
Disallow: /cms/en/careeradvice/event/2008/
Disallow: /cms/en/careeradvice/event/2009/
Disallow: /cms/en/careeradvice/event/2010/
Disallow: /cms/en/careeradvice/event/2011/
Disallow: /cms/en/recruiter/event/2007/
Disallow: /cms/en/recruiter/event/2008/
Disallow: /cms/en/recruiter/event/2009/
Disallow: /cms/en/recruiter/event/2010/
Disallow: /cms/en/recruiter/event/2011/
Disallow: /cnl/
Disallow: /recipes/
Disallow: /*?recipe_id
Disallow: /*cobrand_landing.php
Disallow: /profiles/
Disallow: /profile/
Allow: /profiles/blog/list



# OFonseca- slurp and msn were a bit aggressive on jan 19, 2010

User-agent: Slurp
Crawl-delay: 05

User-agent: msnbot
Crawl-delay: 10


# OFonseca- possible bad/feisty crawlers feb 2, 2009

User-agent: CCBot
Crawl-delay: 10

User-agent: Gigabot
Crawl-delay: 10

User-agent: Jambot
Crawl-delay: 10

User-agent: Teoma
Crawl-delay: 10

User-agent: Twiceler
Crawl-delay: 10

User-agent: Speedy
Crawl-Delay: 10

# Bots with bad rep

User-agent: YangaWorldSearch
Disallow: / 

User-Agent: VoilaBot
Disallow: / 

User-agent: Exabot
Crawl-delay: 10

User-agent: Twiceler
Crawl-delay: 10