# /robots.txt file for http://www.clandarkmatter.co.uk/ User-agent: * Disallow: /info.php Disallow: /news.php Disallow: /ts.php User-agent: Slurp Crawl-delay: 20 User-agent: msnbot Crawl-delay: 86400 #delay parameter - how often, in seconds