# Robot exclusion file
# See http://info.webcrawler.com/mak/projects/robots/exclusion.html


# Temporarily set for local Netscape Catalog server
User-agent: Netscape-Catalog-Robot/1.01
Disallow: /Harvest/
Disallow: /cgi-bin/
Disallow: /dai/
Disallow: /daidb/conduits/
Disallow: /daidb/design
Disallow: /daidb/grg/
Disallow: /daidb/lists/
Disallow: /daidb/people/portraits/
Disallow: /daidb/people/homes/
Disallow: /daidb/reports/
Disallow: /daidb/staff/
Disallow: /daidb/staff-only/
Disallow: /daidb/students/
Disallow: /daidb/testing/
Disallow: /ed/
Disallow: /groups/evalg/Local_Copies_of_Papers/
Disallow: /groups/evalg/Process_local_copies/
Disallow: /groups/nlp/nlp/
Disallow: /html/
Disallow: /staff/
Disallow: /stats/
Disallow: /students/
Disallow: /ito/

# List of initial URLs for our harvest gatherers to skip
# DAI/Harvest/1.5 is a 'special' pattern to match against the User-Agent
# value set for our gatherers
User-agent: DAI/Harvest/1.5
Disallow: /Harvest/
Disallow: /cgi-bin/
Disallow: /dai/staff_only/
Disallow: /daidb/conduits/
Disallow: /daidb/staff-only/
Disallow: /daidb/testing/


# Default for robots not mentioned elsewhere
User-agent: *
Disallow: /Harvest/
Disallow: /cgi-bin/
Disallow: /dai/
Disallow: /daidb/conduits/
Disallow: /daidb/design
Disallow: /daidb/grg/
Disallow: /daidb/lists/
Disallow: /daidb/people/portraits/
Disallow: /daidb/people/homes/
Disallow: /daidb/reports/
Disallow: /daidb/staff/
Disallow: /daidb/staff-only/
Disallow: /daidb/students/
Disallow: /daidb/testing/
Disallow: /ed/
Disallow: /groups/evalg/Local_Copies_of_Papers/
Disallow: /groups/evalg/Process_local_copies/
Disallow: /groups/nlp/nlp/
Disallow: /html/
Disallow: /staff/
Disallow: /stats/
Disallow: /students/
Disallow: /ito/