diff options
Diffstat (limited to 'webapp/static/robots.txt')
| -rw-r--r-- | webapp/static/robots.txt | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/webapp/static/robots.txt b/webapp/static/robots.txt new file mode 100644 index 0000000000..c0339176af --- /dev/null +++ b/webapp/static/robots.txt @@ -0,0 +1,8 @@ +# Directions for web crawlers. +# See http://www.robotstxt.org/wc/norobots.html. + +User-agent: HTTrack +User-agent: puf +User-agent: MSIECrawler +User-agent: Nutch +Disallow: / |