Некоторые боты в файле bots_settings.php из "Скрипт статистики посещений поисковыми ботами на PHP" и файл robots.txt

// Known bots list

$bot_names['ABCdatos'] = 'BotLink';
$bot_names['Acme.Spider'] = 'acme-spider';
$bot_names['Addon firefox'] = 'Pearltrees';
$bot_names['Adventure Bot'] = 'AdvBot';
$bot_names['Ahoy!'] = 'ahoy';
$bot_names['Ahrefs Bot'] = 'AhrefsBot';
$bot_names['Alexa'] = 'ia_archiver';
$bot_names['Alkaline'] = 'AlkalineBOT';
$bot_names['ALTWeb Capital'] = 'MegaIndex.ru';
$bot_names['AltaVista'] = 'Scooter';
$bot_names['Amazon Bitly'] = 'bitlybot';
$bot_names['Amazon DDG'] = 'DuckDuckGo-Favicons-Bot';
$bot_names['Amazon Jivo'] = 'Jivobot';
$bot_names['Anthill'] = 'anthill';
$bot_names['Aport'] = 'Aport';
$bot_names['Arachnophilia'] = 'Arachnophilia';
$bot_names['Arale'] = 'arale';
$bot_names['Araneo'] = 'araneo';
$bot_names['AraybOt'] = 'AraybOt';
$bot_names['ArchitextSpider'] = 'ArchitextSpider';
$bot_names['Aretha'] = 'aretha';
$bot_names['Archive.org'] = 'archive.org';
$bot_names['Ariadne'] = 'ariadne';
$bot_names['arks'] = 'arks';
$bot_names['Ask'] = 'Teoma';
$bot_names['AskJeeves'] = 'Jeeves';
$bot_names['Associative Spider'] = 'ASpider';
$bot_names['ATN Worldwide'] = 'ATN_Worldwide';
$bot_names['Atomz.com'] = 'Atomz';
$bot_names['Auresys'] = 'AURESYS';
$bot_names['Bad links'] = 'Xenu Link Sleuth';
$bot_names['Baidu'] = 'Baiduspider';
$bot_names['Begun'] = 'BegunAdvertising';
$bot_names['Bingbot'] = 'bingbot';
$bot_names['BLEXBot Crawler'] = 'BLEXBot';
$bot_names['BlogPulse'] = 'BlogPulse';
$bot_names['BlogScope'] = 'BlogScope';
$bot_names['Boomerang'] = 'Boomerang';
$bot_names['City Market'] = 'occbot';
$bot_names['CJSC Tech. centre'] = 'statdom.ru';
$bot_names['CloudFlare Inc'] = 'MediaLBot';
$bot_names['Crawler'] = 'crw';
$bot_names['DobroBot'] = 'DobroBot';
$bot_names['ELinks'] = 'ELinks';
$bot_names['Exalead'] = 'Exabot';
$bot_names['Facebook'] = 'facebookexternalhit';
$bot_names['Feed Burner'] = 'FeedBurner';
$bot_names['FollowSite'] = 'FollowSite Bot';
$bot_names['Format-tv ua'] = 'MediaBot';
$bot_names['FriendFeed'] = 'FriendFeedBot';
$bot_names['Genieo'] = 'Genieo';
$bot_names['Getdownload.ws'] = 'DBot';
$bot_names['Gigabot'] = 'Gigabot';
$bot_names['GoldenSpider'] = 'GoldenSpider';
$bot_names['Google AdsBot'] = 'AdsBot-Google';
$bot_names['GoogleAdsense'] = 'Mediapartners';
$bot_names['Google Adsense'] = 'Mediapartners-Google';
$bot_names['Google Favicon'] = 'Google favicon';
$bot_names['Google Images'] = 'Googlebot-Image';
$bot_names['Google Mobile'] = 'Googlebot-Mobile';
$bot_names['Google News'] = 'Googlebot-News';
$bot_names['Google Verification'] = 'Google-Site-Verification';
$bot_names['Google Video'] = 'Googlebot-Video';
$bot_names['Google'] = 'Googlebot';
$bot_names['Grapeshot'] = 'GrapeshotCrawler';
$bot_names['helsinki.fi'] = 'sukibot_heritrix';
$bot_names['Hospital HunChan'] = 'lechenie mezhpozvonochnoi grizhi';
$bot_names['HostGator Inc'] = 'GoogleHttpClient';
$bot_names['Huasai'] = 'Huasai';
$bot_names['iGde'] = 'igdeSpyder';
$bot_names['Internet Memory org'] = 'memorybot';
$bot_names['iParadigms Inc'] = 'TurnitinBot';
$bot_names['JSC Rostovelectrosviaz'] = 'Konqueror';
$bot_names['la.net.ua'] = 'Pizilla';
$bot_names['LeechCraft'] = 'LeechCraft';
$bot_names['Libra [China]'] = 'librabot';
$bot_names['Lightspeed Tech.'] = 'LSSRocketCrawler';
$bot_names['Linkdex Limited'] = 'linkdexbot';
$bot_names['Linkpad'] = 'LinkpadBot';
$bot_names['LiveInternet'] = 'LiveInternet';
$bot_names['Lynx'] = 'Lynx';
$bot_names['Lycos'] = 'Lycos';
$bot_names['Mail.Ru'] = 'Mail.Ru';
$bot_names['Majestic-12'] = 'MJ12bot';
$bot_names['Midori'] = 'Midori';
$bot_names['Meanpath'] = 'meanpathbot';
$bot_names['Mir Telematiki'] = 'WebIndex';
$bot_names['MirTelematiki'] = 'MirTelematikiWebIndex';
$bot_names['MSNBot Academic'] = 'msnbot-academic';
$bot_names['MSNbot Media'] = 'msnbot-media';
$bot_names['MSNBot NewsBlogs'] = 'msnbot-newsblogs';
$bot_names['MSNBot Products'] = 'msnbot-products';
$bot_names['MSN'] = 'msnbot';
$bot_names['NetCraft'] = 'NetcraftSurveyAgent';
$bot_names['Nuance communications'] = 'lmspider';
$bot_names['Odnoklassniki'] = 'OdklBot';
$bot_names['OMG Crawler'] = 'OMGCrawler';
$bot_names['Omni Group'] = 'OmniWeb';
$bot_names['OozBot'] = 'OOZBOT';
$bot_names['Open Site Exploler'] = 'DotBot';
$bot_names['-Openstat-'] = 'openstat.ru';
$bot_names['Ovale'] = 'ovalebot';
$bot_names['Poker bot'] = 'Moreover';
$bot_names['PostRank'] = 'PostRank';
$bot_names['Pr-cy.ru'] = 'pr-cy.ru';
$bot_names['Prlog seo'] = 'Prlog';
$bot_names['Proximic Bot'] = 'proximic';
$bot_names['PSINet Inc'] = 'Qwantify';
$bot_names['Pulsepoint Inc'] = 'ContextAd';
$bot_names['Rambler'] = 'StackRambler';
$bot_names['Semrush'] = 'SemrushBot';
$bot_names['Sistrix'] = 'SISTRIX Crawler';
$bot_names['SoftLayer Tech. Inc'] = 'WinHTTP';
$bot_names['Steam'] = 'SMTBot';
$bot_names['Snap Shots'] = 'Snapbot';
$bot_names['Space bison'] = 'Space Bison';
$bot_names['stat.reg.ru'] = 'StatOnlineRuBot';
$bot_names['System Monitoring Web'] = 'Web-Monitoring';
$bot_names['Tagoo'] = 'Tagoobot';
$bot_names['Tele House'] = 'Dolphin';
$bot_names['Twiceler'] = 'Twiceler';
$bot_names['Twingly'] = 'Twingly Recon';
$bot_names['Twitter'] = 'Twitterbot';
$bot_names['URL Grabber'] = 'URLGrabber';
$bot_names['W3C Mobile'] = 'W3C-mobileOK';
$bot_names['W3C Validator'] = 'Validator';
$bot_names['W3 Sitesearch'] = 'W3 SiteSearch Crawler';
$bot_names['WebAlta'] = 'WebAlta';
$bot_names['WebAlta crw'] = 'WebAlta Crawler';
$bot_names['Wget Spam'] = 'Wget';
$bot_names['Woobot'] = 'woobot';
$bot_names['WordPress'] = 'WordPress';
$bot_names['Worio'] = 'woriobot';
$bot_names['Wortmann Ag'] = 'securepoint';
$bot_names['Wotbox'] = 'Wotbox';
$bot_names['Yahoo!'] = 'Slurp';
$bot_names['Yahoo! Slurp'] = 'Yahoo! Slurp';
$bot_names['Yandex'] = 'YandexBot';
$bot_names['Yandex Blogs'] = 'YandexBlogs';
$bot_names['Yandex Direct'] = 'YandexDirect';
$bot_names['Yandex Favicons'] = 'YandexFavicons';
$bot_names['Yandex Images'] = 'YandexImages';
$bot_names['Yandex Market'] = 'YandexMarket';
$bot_names['Yandex Media'] = 'YandexMedia';
$bot_names['Yandex Net'] = 'Go';
$bot_names['Yandex Webmaster'] = 'YandexWebmaster';
$bot_names['Yandex Video'] = 'YandexVideo';
$bot_names['Yanga'] = 'Yanga WorldSearch';
$bot_names['Youdao'] = 'YoudaoBot';
$bot_names['Your-server de 4'] = 'Links';
$bot_names['Virusdie'] = 'Virusdie crawler';
$bot_names['VkShare'] = 'vkShare';

// You can add new bots here in the format above.

User-agent: Aport
User-agent: BegunAdvertising
User-agent: bingbot
User-agent: FeedBurner
User-agent: Go
User-agent: Google favicon
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: Google-Site-Verification
User-agent: Googlebot-Video
User-agent: Mediapartners
User-agent: Mediapartners-Google
User-agent: msnbot
User-agent: msnbot-media
User-agent: msnbot-newsblogs
User-agent: msnbot-products
User-agent: LiveInternet
User-agent: StackRambler
User-agent: Teoma
User-agent: YandexBot
User-agent: YandexBlogs
User-agent: YandexDirect
User-agent: YandexFavicons
User-agent: YandexImages
User-agent: YandexMarket
User-agent: YandexMedia
User-agent: YandexWebmaster
User-agent: YandexVideo
Disallow: /administrator/
Disallow: /cache/
Disallow: /cli/
Disallow: /components/
Disallow: /images/
Disallow: /includes/
Disallow: /language/
Disallow: /libraries/
Disallow: /logs/
Disallow: /media/
Disallow: /modules/
Disallow: /plugins/
Disallow: /templates/
Disallow: /tmp/
Host: https://myhouselife.ru:443
Sitemap: https://myhouselife.ru/sitemap.xml