User-agent: * disallow: /emailProduct.asp disallow: /admin/ disallow: /awstats/ disallow: /styles/ disallow /pages/ProductdetailsContent.asp # disallow panscient.com as they try to crawl invalid directories User-agent: panscient.com disallow: / # disallow botmobi as they try to crawl invalid directories User-agent: botmobi disallow: / # disallow Twiceler as they ignore directives - let's see if they follow this one... User-agent: Mozilla/5.0+(Twiceler-0.9+http://www.cuil.com/twiceler/robot.html) disallow: / # disallow MJ12bot as they ignore directives - let's see if they follow this one... User-agent: Mozilla/5.0+(compatible;+MJ12bot/v1.3.1;+http://www.majestic12.co.uk/bot.php?+) User-agent: Mozilla/5.0+(compatible;+MJ12bot/v1.3.1;+http://www.majestic12.co.uk/bot.php?+) disallow: /