# /robots.txt file for http://yourwebsite.com/

# Example below allows access to entire site for crawler1
# User-agent: crawler1
# Disallow:

# Example below disallows access to entire site for crawler2
# User-agent: crawler2
# Disallow: /

# Example below disallows access to any other user agent for the tmp directory and oldfile.html
# User-agent: *
# Disallow: /tmp
# Disallow: /oldfile.html