########### Robots ######
# This would tell all spiders that you don't want them on your site at all.
#


# All robots will spider the domain
User-agent: *
Crawl-Delay: 10
Disallow: /archive
Disallow: /user
Disallow: /admin
Disallow: /groups

#
# To tell them not to go to only certain parts of your site change the Disallow
# line to list those parts, always starting the directory name with a slash.
#
# eg: Disallow: /cgi-bin/
#
# You can exclude other directories by listing them on more Disallow lines below
# the first one. To exclude a particular spider put that spider's name on the
# User-agent: line instead of the *.