# All robots will spider the domain User-agent: * Disallow: # Disallow the following User-agent: * Disallow: /images/ Disallow: /sendemail.php Disallow: /sendenergy.php Disallow: /flash/ Disallow: /pdfs/