# File to stop web-crawlers indexing this directory
# or any sub-directories
# v1.00 Rich Mellor 10/6/04

# Sample to stop NorthcoteSpider from crawling the website
# except for information in info folder or files beginning info, eg. info.html
User-agent: northcotespider
Allow: /info
Disallow: /

# Allow all other robots complete access
User-agent: *
Allow: /