# EXAMPLE robots.txt file - see http://www.robotstxt.org/wc/norobots.html User-agent: * Disallow: /tmp/ # these will soon disappear, don't index anything in this directory Disallow: /robotstxtorg.html # don't index the file robotstxtorg.html #block alexa http://pages.alexa.com/help/webmasters/#prevent #http://web.archive.org/collections/web.html User-agent: ia_archiver Disallow: /