software/!RELEASES/ircservices/achurch.org/robots.txt

   1 # msnbot misbehaves
   2 User-agent: msnbot
   3 Disallow: /
   4
   5 # Block SEO bots, which generally misbehave
   6 User-agent: AhrefsBot
   7 Disallow: /
   8 User-agent: AlexiBot
   9 Disallow: /
  10 User-agent: DataForSeoBot
  11 Disallow: /
  12 User-agent: dotbot
  13 Disallow: /
  14 User-agent: MJ12bot
  15 Disallow: /
  16 User-agent: rogerbot
  17 Disallow: /
  18 User-agent: SemrushBot
  19 Disallow: /
  20 User-agent: SemrushBot-SA
  21 Disallow: /
  22 User-agent: SurveyBot
  23 Disallow: /
  24 User-agent: Xenu
  25 Disallow: /
  26 User-agent: Xenu Link Sleuth
  27 Disallow: /
  28
  29 # Let wget in to get public stuff
  30 User-agent: wget
  31 Disallow: /priv/
  32
  33 # Googlebot is well-behaved, so it can access Mercurial repos
  34 User-agent: Googlebot
  35 Disallow: /priv/
  36 Disallow: /cgi-bin/yomiuri
  37 Disallow: /landscape-pre.gif
  38 Disallow: /stories/l
  39
  40 # Everything else
  41 User-agent: *
  42 Disallow: /priv/
  43 Disallow: /cgi-bin/yomiuri
  44 Disallow: /landscape-pre.gif
  45 Disallow: /stories/l
  46 # Avoid badly designed bots overloading the server
  47 Disallow: /hg