# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: archive.org_bot
Allow:       /   
Disallow: /code/
Disallow: /Test/
Disallow: /cgi-bin/
Disallow: /doc/Contact_Email.asp
Disallow: /brd/BRD_contact_email.asp
Disallow: /COMDEV/CDD_Contact_Email.asp
Disallow: /housing/Hous_Contact_email.asp
Disallow: /MHC/MHC/Contact_email.asp
Disallow: /doc/Contact.asp
Disallow: /brd/BRD_contact.asp
Disallow: /COMDEV/CDD_Contact.asp
Disallow: /housing/Hous_Contact.asp
Disallow: /MHC/MHC/Contact.asp
Disallow: /Net Tracker/
Disallow: /Templates/
Disallow: /scripts/
Disallow: /Commerce Folder/
Disallow: /ceic/

User-agent: webcrawler
Disallow: /includes/
Disallow: /images/
Disallow: /code/
Disallow: /Test/
Disallow: /cgi-bin/
Disallow: /doc/Contact.asp
Disallow: /brd/BRD_contact.asp
Disallow: /COMDEV/CDD_Contact.asp
Disallow: /housing/Hous_Contact.asp
Disallow: /MHC/MHC/Contact.asp
Disallow: /doc/Contact_Email.asp
Disallow: /brd/BRD_contact_email.asp
Disallow: /COMDEV/CDD_Contact_Email.asp
Disallow: /housing//Hous_Contact_email.asp
Disallow: /MHC/MHC/Contact_email.asp
Disallow: /Net Tracker/
Disallow: /Templates/
Disallow: /scripts/
Disallow: /Commerce Folder/
Disallow: /ceic/

User-agent: lycra
Disallow: /includes/
Disallow: /images/
Disallow: /code/
Disallow: /Test/
Disallow: /cgi-bin/
Disallow: /doc/Contact.asp
Disallow: /brd/BRD_contact.asp
Disallow: /COMDEV/CDD_Contact.asp
Disallow: /housing/Hous_Contact.asp
Disallow: /MHC/MHC/Contact.asp
Disallow: /doc/Contact_Email.asp
Disallow: /brd/BRD_contact_email.asp
Disallow: /COMDEV/CDD_Contact_Email.asp
Disallow: /housing//Hous_Contact_email.asp
Disallow: /MHC/MHC/Contact_email.asp
Disallow: /Net Tracker/
Disallow: /Templates/
Disallow: /scripts/
Disallow: /Commerce Folder/
Disallow: /ceic/

User-agent: *
Disallow: /includes/
Disallow: /images/
Disallow: /code/
Disallow: /Test/
Disallow: /cgi-bin/
Disallow: /doc/Contact.asp
Disallow: /brd/BRD_contact.asp
Disallow: /COMDEV/CDD_Contact.asp
Disallow: /housing/Hous_Contact.asp
Disallow: /MHC/MHC/Contact.asp
Disallow: /doc/Contact_Email.asp
Disallow: /brd/BRD_contact_email.asp
Disallow: /COMDEV/CDD_Contact_Email.asp
Disallow: /housing//Hous_Contact_email.asp
Disallow: /MHC/MHC/Contact_email.asp
Disallow: /Net Tracker/
Disallow: /Templates/
Disallow: /scripts/
Disallow: /Commerce Folder/
Disallow: /ceic/