# Robots.txt file for https://mississaugarealestate.ca
#
# 1) Do not allow Internet Archive (WayBackMachine)
#    (Redundant, as should be excluded per 3 below)
#
# 2) Allow Specific Search Engine Bots only
#
#   Google Crawlers: https://support.google.com/webmasters/answer/1061943?hl=en
#
#   Bing Crawlers: https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
#
#   Twitter Crawlers: https://dev.twitter.com/cards/getting-started
#		      https://cards-dev.twitter.com/validator
#   		     (Req'd to allow Twitter Cards to be generated)
#
# 3) Do not allow any other bots (used wildcard)

User-agent: ia_archiver
Disallow: /

User-agent: Googlebot
User-agent: Google
User-agent: Googlebot-News
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-Mobile
User-agent: Mediapartners-Google
User-agent: Mediapartners-Google*
User-agent: Mediapartners
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile-Apps
User-agent: Bingbot
User-agent: MSNBot
User-agent: MSNBot-Media
User-agent: AdIdxBot
User-agent: BingPreview
User-agent: Slurp
User-agent: Twitterbot
Disallow: 

User-agent: *
Disallow: /