# robots.txt for library.duke.edu          
# Contact library-digital-projects@duke.edu 
                                           
# All bots
User-agent: *
Disallow: /digitalcollections/?DwebQuery=       # Dynaweb
Disallow: /digitalcollections/*?DwebQuery=      # Dynaweb
Disallow: /digitalcollections/?facet=
Disallow: /digitalcollections/?*&facet=
Disallow: /digitalcollections/*?facet=
Disallow: /digitalcollections/*?*&facet=
Disallow: /digitalcollections/*/?q=
Disallow: /digitalcollections/grid/
Disallow: /digitalcollections/images/
Disallow: /digitalcollections/imgs/
Disallow: /digitalcollections/list/
Disallow: /digitalcollections/pdf/
Disallow: /digitalcollections/sizes/
Disallow: /digitalcollections/wall/
Disallow: /digitalcollections/*/search/results? # Tripod 1
Disallow: /digitalcollections/Date/             # RT #191641
Disallow: /digitalcollections/Spatial_Coverage/ # RT #191631
Disallow: /digitalcollections/Subject/          # RT #191611
Disallow: /librarycatalog
Disallow: /rubenstein/findingaids/*.xml         # XML finding aids, RT #229021
Disallow: /rubenstein/findingaids/*.pdf         # PDF finding aids
# end all bots

User-agent: Googlebot
Disallow: /digitalcollections/?DwebQuery=       # Dynaweb
Disallow: /digitalcollections/*?DwebQuery=      # Dynaweb
Disallow: /digitalcollections/?facet=
Disallow: /digitalcollections/?*&facet=
Disallow: /digitalcollections/*?facet=
Disallow: /digitalcollections/*?*&facet=
Disallow: /digitalcollections/*/?q=
Disallow: /digitalcollections/grid/
Disallow: /digitalcollections/images/
Disallow: /digitalcollections/imgs/
Disallow: /digitalcollections/list/
Disallow: /digitalcollections/pdf/
Disallow: /digitalcollections/sizes/
Disallow: /digitalcollections/wall/
Disallow: /digitalcollections/*/search/results? # Tripod 1
Disallow: /digitalcollections/Date/             # RT #191641
Disallow: /digitalcollections/Spatial_Coverage/ # RT #191631
Disallow: /digitalcollections/Subject/          # RT #191611
Disallow: /librarycatalog
Disallow: /rubenstein/findingaids/*.xml         # XML finding aids, RT #229021
Allow: /digitalcollections/media/jpg/		# RT 253331 - 1/9/13 dcs

User-agent: Googlebot-Image
Disallow: /digitalcollections/
Allow: /digitalcollections/media/jpg/		# RT 253331 - 1/9/13 dcs

# http://www.baidu.com/search/spider.html - Added 5/31/11
User-agent: Baiduspider
Disallow: /digitalcollections/

# http://www.80legs.com/spider.html
User-agent: 008
Disallow: /digitalcollections/

# http://ws.daum.net/aboutWebSearch_en.html
User-agent: Daumoa
Disallow: /digitalcollections/

User-agent: Ezooms
Disallow: /

# http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
User-agent: MJ12bot
Disallow: /digitalcollections/

# Added 5/26/2011 - http://www.scirus.com/srsapp/contactus/
User-agent: Scirus
Disallow: /digitalcollections/

# Added 5/26/2011 - http://www.entireweb.com/about/search_tech/speedy_spider/
User-agent: Speedy
Disallow: /digitalcollections/

# Yahoo! slurp
User-agent: Slurp
Crawl-delay: 2