User-agent: * # match all bots. The Google Search Appliance (GSA) is our primary crawler but logs indicate there may be others on our Intranet
Crawl-delay: 10 # per http://en.wikipedia.org/wiki/Robots.txt#Nonstandard_extensions, sets number of seconds to wait between requests to 5 seconds. may not work
Request-rate: 1/10 # per http://en.wikipedia.org/wiki/Robots.txt#Extended_Standard, maximum rate is one page every 5 seconds. may not work
# DISABLED FOR NOW Visit-time: 0600-0845 # per http://en.wikipedia.org/wiki/Robots.txt#Extended_Standard, only visit between 6:00 AM and 8:45 AM UT (GMT), may not work
Visit-time: 0400-0900

Disallow: /displayFile2.jsp* # don't let any bot try to display a file
Disallow: /downloadFile.jsp* # don't let any bot try to download large media

User-agent: dotbot
Disallow: /