User-agent:*

Disallow:/cgi-bin/

Disallow:/error_docs/

Disallow:/pd/

Disallow:/statistics/

Disallow:/subdomains/

Disallow;/web_users/

Disallow:/bin/

Disallow:/etc/

Disallow:/lib/

Disallow:/private/

Disallow:/tmp/

Disallow:/usr/

Disallow:/var/

Disallow:/httpdocs/_private/

Disallow:/httpdocs/backup/

Disallow:/httpdocs/catalog/

Disallow:/httpdocs/Scripts/

Disallow:/httpdocs/_vti_log/

Disallow:/httpdocs/awstats/

Disallow:/httpdocs/log/

Disallow:/httpdocs/customerrors/

Disallow:/httpdocs/backup/

Disallow:/httpdocs/_vti_bin/

Disallow:/httpdocs/_vti_cnf/

Disallow:/httpdocs/_vti_pvt/

Disallow:/httpdocs/_vti_txt/

almaden.ibm.com

appie 1.1

architext

ask jeeves

asterias2.0

augurfind

baiduspider

bannana_bot

bdcindexer

crawler

crawler@fast

docomo

fast-webcrawler

fluffy the spider

frooglebot

geobot

googlebot

gulliver

henrythemiragorobot

ia_archiver

infoseek

kit_fireball

lachesis

lycos_spider

mantraagent

mercator

moget/1.0

muscatferret

nationaldirectory-webspider

naverrobot

ncsa beta

netresearchserver

ng/1.0

osis-project

polybot

pompos

scooter

seventwentyfour

sidewinder

sleek spider

slurp/si

slurp@inktomi.com

steeler/1.3

szukacz

t-h-u-n-d-e-r-s-t-o-n-e

teoma

turnitinbot

ultraseek

vagabondo

voilabot

w3c_validator

zao/0

zyborg/1.0

MJ12bot

Yandex

Netcraft

psbot

SurveyBot

Yahoo! Slurp China

Nutch

BSpider

Exabot