#This tells all crawlers not to enter admin directory:
#addresses all robots by using wild card *

User-agent: *
Disallow: /admin/
Disallow: /catalog/checkout_confirmaton.php
Disallow: /catalog/checkout_success.php
Disallow: /catalog/advanced_search_result.php
Disallow: /catalog/advanced_search.php
# maximum rate is one page every 120 seconds
#Request-rate: 1/120       

#should only visit between 01:00 am and 05:30am
Visit-time: 0130-500  

User-agent: Slurp
Crawl-delay: 20
Disallow: /admin/
Disallow: /catalog/checkout_confirmaton.php
Disallow: /catalog/checkout_success.php
Disallow: /catalog/advanced_search_result.php
Disallow: /catalog/advanced_search.php
# maximum rate is one page every 120 seconds
Request-rate: 1/120       

#should only visit between 01:00 am and 05:30am
Visit-time: 0130 '-' 500