User-agent: *
Allow: / 

almaden.ibm.com
appie 1.1
architext
ask jeeves
asterias2.0
augurfind
baiduspider
bannana_bot
bdcindexer
crawler
crawler@fast
docomo
fast-webcrawler
fluffy the spider
frooglebot
geobot
googlebot
poodle
gulliver
henrythemiragorobot
ia_archiver
infoseek
kit_fireball
lachesis
lycos_spider
mantraagent
mercator
moget/1.0
muscatferret
nationaldirectory-webspider
naverrobot
ncsa beta
netresearchserver
ng/1.0
osis-project
polybot
pompos
scooter
seventwentyfour
sidewinder
sleek spider
slurp/si
slurp@inktomi.com
steeler/1.3
szukacz
t-h-u-n-d-e-r-s-t-o-n-e
teoma
turnitinbot
ultraseek
vagabondo
voilabot
w3c_validator
zao/0
zyborg/1.0
crawl
slurp
spider
ebot
obot
abot
dbot
hbot
nbot
pbot
rbot
sbot
ybot
accoona
appie
architext
aspseek
asterias
atlocal
atomz
augurfind
bannana_bot
booch
docomo
gazz
goforit
grub
gulliver
harvest
holmes
homer
ia_archiver
ichiro
iconsurf
iltrovatore
indexer
infoseek
ivia
java/
jetbot
kit_fireball
knowledge
lachesis
larbin
linkwalker
lwp
mantraagent
mediapartners
mercator
mj12
moget/
msnbot
multitext
muscatferret
myweb
nameprotect
ncsa beta
netmechanic
netresearchserver
ng/
npbot
nutch
objectssearch
osis-project
pear.
pompos
poppelsdorf
rambler
scooter
scrubby
seeker
shopwiki
sidewinder
smartwit
sohu
spyder
steeler/
szukacz
t-h-u-n-d-e-r-s-t-o-n-e
/teoma
tutorgig
ultraseek
vagabondo
volcano
voyager/
w3c_validator
webcheck
websitepulse
wget
worldlight
worm
zao/
xenu
zippp
zyborg

# CRELoaded Generated Robots.txt
# Robot Exclusion File -- robots.txt
# Author:  CRELoaded Team 
# Last Updated : September 11th, 2007

User-Agent: *
Disallow: /admin/
Disallow: /cache/
Disallow: /cert/
Disallow: /debug/
Disallow: /download/
Disallow: /ext/
Disallow: /includes/
Disallow: /pear/
Disallow: /pub/
Disallow: /temp/
Disallow: /templates/
Disallow: /tmp/


#To prevent only a select bots from any scans
#User-agent: crawlers.looksmart.com
#Disallow: /


#To prenet any images from being scanned
#Disallow: *.gif
#Disallow: *.jpg
#Disallow: *.png


#To block main page due to size from bots
#Disallow: /index.php


#To shut down site completely from bots use
# Disallow: /


#Block out things that are secure or login oriented
#These are based on not using any rewrite rules
Disallow: /account.php
Disallow: /account_edit.php
Disallow: /account_history.php
Disallow: /account_history_info.php
Disallow: /account_password.php
Disallow: /address_book.php
Disallow: /address_book_process.php
Disallow: /checkout_confirmation.php
Disallow: /checkout_payment.php
Disallow: /checkout_payment_address.php
Disallow: /checkout_process.php
Disallow: /checkout_shipping.php
Disallow: /checkout_shipping_address.php
Disallow: /checkout_success.php
Disallow: /cookie_usage.php
Disallow: /create_account.php
Disallow: /create_account_success.php
Disallow: /ipn.php
Disallow: /login.php
Disallow: /logoff.php
Disallow: /password_forgotten.php
Disallow: /paypal_notify.php
Disallow: /redirect.php
Disallow: /shipping_estimator_popup.php
Disallow: /ssl_check.php
Disallow: /vvc_display.php
Disallow: /wpcallback.php


# IF YOU DO NOT WISH TO HAVE THE GOOGLE IMAGE BOT SCAN YOUR DOMAIN FOR IMAGES
# THEN YOU CAN INCLUDE THE FOLLOWING IN YOUR ROBOTS FILE.
# I FOUND THAT MY BANDWIDTH USAGE DROPPED BY A MASSIVE AMOUNT AFTER I GOT RID
# OF THE GOOGLE IMAGE BOT. ALL I HAD WAS IMAGE HUNTERS STEALING PRODUCT SHOTS
# AND NOT EVEN BROWSING THE SITE.
#User-agent: Googlebot-Image
#Disallow: /