# /robots.txt for http://www.webhostingsizzle.com/
# Search engine: "norobots.html"
# Admin - Elijah  (mhidelivers.com/feedback)

# disallow a bunch of ill-behaved user agents (doubt this will deter them...)

User-agent: TurnitinBot
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf 
Disallow: /

User-agent: CherryPickerSE/1.0
Disallow: /

User-agent: CherryPickerElite/1.0
Disallow: /

User-agent: EmailCollector/1.0
Disallow: /

User-agent: EmailWolf 1.00
Disallow: /

User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: Mozilla/2.0 (compatible; NEWT ActiveX; Win32)
Disallow: /


# let google's ad crawler do its thing
User-agent: Mediapartners-Google*
Disallow: 
Disallow: /rate		# dynamic
Disallow: /cgi-bin	# dynamic
Disallow: /search	# dynamic

# by default

User-agent: *
Disallow: /ccrd			# not useful to spiders
Disallow: /click		# not useful to spiders
Disallow: /hc			# dynamic
Disallow: /rate			# dynamic
Disallow: /subbatch		# dynamic
Disallow: /vadd			# dynamic
Disallow: /vsearch		# dynamic
Disallow: /search		# dynamic
Disallow: /vedit		# dynamic
Disallow: /vdelete		# dynamic
Disallow: /cgi-bin		# dynamic
Disallow: /images/		# useless images
Disallow: /icons/		# useless images
Disallow: /concerts/		# deprecated URL form
Disallow: /conferences		# defunct

#Examples
#The following example "/robots.txt" file specifies that no robots should visit any URL starting with "/cyberworld/map/" or 
"/tmp/", or /foo.html: 
#------------------------------------------------------------------------


# robots.txt for http://www.example.com/

#User-agent: *
#Disallow: /cyberworld/map/ # This is an infinite virtual URL space
#Disallow: /tmp/ # these will soon disappear
#Disallow: /foo.html

#------------------------------------------------------------------------
#This example "/robots.txt" file specifies that no robots should visit any URL starting with "/cyberworld/map/", except the 
robot called "cybermapper": 
#------------------------------------------------------------------------


# robots.txt for http://www.example.com/

#User-agent: *
#Disallow: /cyberworld/map/ # This is an infinite virtual URL space

## Cybermapper knows where to go.
#User-agent: cybermapper
#Disallow:

#------------------------------------------------------------------------
#This example indicates that no robots should visit this site further: 
#------------------------------------------------------------------------


## go away
#User-agent: *
#Disallow: /