# robots.txt for http://www.sfolweb.com
# robots.txt for http://sfolweb.com

user-agent: *  # directed to all spiders, not just Scooter
Disallow: /_dev/
Disallow: /_docs/
Disallow: /art/
Disallow: /banners/
Disallow: /gmt/
Disallow: /images/
Disallow: /nfbooks/
Disallow: /webring/

User-agent: Mediapartners-Google* 
Disallow: 

User-agent: searchpreview
Disallow: /
Hit-rate: 30
# wait 30 seconds before starting a new URL request default=30
Visiting-hours: 01:00EST-05:00EST
# index this site between 1AM - 5AM EST
Concurrent-hits: 2
# limit concurrent active URLs to 2 for each index server