User-agent: * Disallow: /_ mjr/ Disallow: /_borders/ Disallow: /_css/ Disallow: /_private/ Disallow: /_scripts/ Disallow: /2009/ Disallow: /stats/ Disallow: /OldSite_Pre2009/ Disallow: /docs/_secure/ Disallow: /bylawsLogin.htm Disallow: /bylaws.htm # #Non-standard extensions #Crawl-delay: 10 #set to the number of seconds to wait between successive requests to the same server #Allow: #to counteract a following Disallow directive for a specific file or folder #Sitemap: #allowing multiple sitemaps in the same robots.txt in the form # #Extended standard-proposed #Request-rate: 1/5 # maximum rate is one page every 5 seconds #Visit-time: 0600-0845 # only visit between 06:00 and 08:45 UTC (GMT)