User-agent: * Disallow: /*.bmp$ Disallow: /*.axd$ Disallow: */feed/ Disallow: */trackback/ Disallow: /wp-login.php Disallow: /wp-admin/ Disallow: /wp-content/plugins/ Disallow: /wp-includes Disallow: /page-not-found/ Disallow: /scripts/ Disallow: /themes/ Disallow: /languages/ Disallow: /ext/ Disallow: /administrator/ Disallow: /cache/ Disallow: /components/ Disallow: /includes/ Disallow: /installation/ Disallow: /language/ Disallow: /libraries/ Disallow: /media/ Disallow: /modules/ Disallow: /plugins/ Disallow: /templates/ Disallow: /tmp/ Disallow: /xmlrpc/ Disallow: /?s= Disallow: /?sort= Disallow: /?replytocom= Disallow: /?send_to= Disallow: /?c= Disallow: /?r= Disallow: /?post_type= User-agent: 008 #www.80legs.com crawler Disallow: / #block it entirely User-agent: sogou #www.sogou.com crawler Disallow: / #block it entirely User-agent: Synapse #synapse crawler Disallow: / #block it entirely User-Agent: MJ12bot Disallow: User-agent: Python-urllib Disallow: / Reference: http://www.botreports.com/p/python-urllib.shtml Sitemap: http://www.petaasia.com/sitemap_index.xml /* Robots */