robots.txt 

#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips/
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/

File

drupal/robots.txt
View source
  1. #

  2. # robots.txt

  3. #

  4. # This file is to prevent the crawling and indexing of certain parts

  5. # of your site by web crawlers and spiders run by sites like Yahoo!

  6. # and Google. By telling these "robots" where not to go on your site,

  7. # you save bandwidth and server resources.

  8. #

  9. # This file will be ignored unless it is at the root of your host:

  10. # Used:    http://example.com/robots.txt

  11. # Ignored: http://example.com/site/robots.txt

  12. #

  13. # For more information about the robots.txt standard, see:

  14. # http://www.robotstxt.org/wc/robots.html

  15. #

  16. # For syntax checking, see:

  17. # http://www.sxw.org.uk/computing/robots/check.html

  18. 

  19. User-agent: *

  20. Crawl-delay: 10

  21. # Directories

  22. Disallow: /core/

  23. Disallow: /profiles/

  24. # Files

  25. Disallow: /README.txt

  26. Disallow: /web.config

  27. # Paths (clean URLs)

  28. Disallow: /admin/

  29. Disallow: /comment/reply/

  30. Disallow: /filter/tips/

  31. Disallow: /node/add/

  32. Disallow: /search/

  33. Disallow: /user/register/

  34. Disallow: /user/password/

  35. Disallow: /user/login/

  36. Disallow: /user/logout/

  37. # Paths (no clean URLs)

  38. Disallow: /index.php/admin/

  39. Disallow: /index.php/comment/reply/

  40. Disallow: /index.php/filter/tips/

  41. Disallow: /index.php/node/add/

  42. Disallow: /index.php/search/

  43. Disallow: /index.php/user/password/

  44. Disallow: /index.php/user/register/

  45. Disallow: /index.php/user/login/

  46. Disallow: /index.php/user/logout/