# robots.txt file provided by simplemood.com # All robots will spider the domain User-agent: * Disallow: # Disallow directory /images/ User-agent: * Disallow: /images/ # Disallow directory /admin/ User-agent: * Disallow: /admin/ # other settings you may want to use # Disallow file /mystuff/myfile.html # User-agent: * # Disallow: /mystuff/myfile.html # No robot will spider the domain # User-agent: * # Disallow: / # Disallow $ENV{'HTTP_USER_AGENT'} # User-agent: $ENV{'HTTP_USER_AGENT'} # Disallow: /