############################### # # robots.txt file for this website # # addresses all robots by using wild card * User-agent: * Disallow: /cgi-bin/ Disallow: /tmp/ Disallow: /junk/ Disallow: /wp-admin/ User-agent: Yahoo! Slurp Disallow: / User-agent: Googlebot Disallow: / User-agent: survey Disallow: / User-agent: crawl Disallow: / User-agent: Applebot Disallow: / User-agent: YandexBot Disallow: / User-agent: Yahoo! Slurp Disallow: /*?* User-agent: Yahoo! Slurp China Disallow: / User-agent: Pinterestbot Disallow: / User-agent: My Nutch Spider Disallow: / User-agent: A PHP script Disallow: / User-agent: Mediapartners-Google Disallow: / User-agent: Applebot Disallow: / User-agent: yacybot Disallow: / User-agent: Googlebot-Video Disallow: / User-agent: empty user agent string Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: Unknown robot (identified by hit on robots.txt) Disallow: / User-agent: facebookexternalhit Disallow: / User-agent: YandexImages Disallow: / User-agent: TelegramBot Disallow: / User-agent: nutch ( catchall ) Disallow: / User-agent: YandexMobileBot Disallow: / User-agent: YandexMetrika Disallow: / User-agent: DomainStatsBot Disallow: / User-agent: linkchecker Disallow: / User-agent: MSIECrawler Disallow: / User-agent: Lightspeed Disallow: / User-agent: BingPreview Disallow: / User-agent: bidswitchbot Disallow: / User-agent: W3C_Validator Disallow: / User-agent: bingbot Disallow: / User-agent: Validator.nu Disallow: / User-agent: msnbot-media Disallow: / User-agent: Wget Disallow: / User-agent: Microsoft Office Protocol Discovery Disallow: / User-agent: Jigsaw Disallow: / User-agent: Barkrowler Disallow: / User-agent: daum Disallow: / User-agent: YandexBot Disallow: / User-agent: YandexImageResizer Disallow: / User-agent: DuckDuckGo-Favicons-Bot Disallow: / User-agent: DuckDuckBot-Https Disallow: / User-agent: Unknown robot identified by \*bot Disallow: / User-agent: SemrushBot-SI Disallow: / User-agent: archive.org_bot Disallow: / User-agent: DuckDuckBot-Https Disallow: / User-agent: nbot Disallow: / User-agent: bit.ly Disallow: / User-agent: ICC-Crawler Disallow: / User-agent: spider Disallow: / User-agent: survey Disallow: / User-agent: Yahoo! Slurp Disallow: / User-agent: FirePHP Disallow: / User-agent: PhantomJS Disallow: / User-agent: msnbot Disallow: / User-agent: Mediatoolkitbot Disallow: / User-agent: Konqueror Disallow: / User-agent: SeznamBot Disallow: / User-agent: seek Disallow: / User-agent: Apache-HttpClient Disallow: / User-agent: Baidu ( catchall ) Disallow: / User-agent: CUBOT Disallow: / User-agent: Nautic Expo using Firefox/1.5 Disallow: / User-agent: spider Disallow: / User-agent: YisouSpider Disallow: / User-agent: AdsBot-Google Disallow: / User-agent: Ruby script Disallow: / User-agent: semanticbot Disallow: / User-agent: Who.is.Bot Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: ELinks ( Disallow: / User-agent: Barkrowler Disallow: / User-agent: Slackbot-LinkExpanding Disallow: / User-agent: Sogou web spider Disallow: / User-agent: My Nutch Spider Disallow: / User-agent: GetIntent Crawler Disallow: / User-agent: PhantomJS Disallow: / User-agent: Ruby script Disallow: / User-agent: daum Disallow: / User-agent: AdIdxBot Microsoft Ad Quality control Disallow: / User-agent: LinkedInBot Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: YisouSpider Disallow: / User-agent: Barkrowler Disallow: / User-agent: discover Disallow: / User-agent: scrapy Disallow: / User-agent: CCBot Disallow: / User-agent: Konqueror Disallow: / User-agent: MSIECrawler Disallow: / User-agent: bl.uk_lddc_bot Disallow: / User-agent: urllib Disallow: / User-agent: Velen Disallow: / User-agent: Slackbot-LinkExpanding Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: 360Spider Disallow: / User-agent: Baiduspider Disallow: / User-agent: yak-linkfluence Disallow: / User-agent: Ruby script Disallow: / User-agent: Wget Disallow: / User-agent: semanticbot Disallow: / User-agent: Sogou web spider Disallow: / User-agent: Baiduspider Disallow: / User-agent: FirePHP Disallow: / User-agent: hit Disallow: / User-agent: bot\* Disallow: / User-agent: Velen Disallow: / User-agent: Wget Disallow: / User-agent: seek Disallow: / User-agent: archive.org_bot Disallow: / User-agent: CFNetwork Disallow: / User-agent: PhantomJS Disallow: / User-agent: Mail.RU Bot Disallow: / User-agent: bidswitchbot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: bidswitchbot Disallow: / User-agent: Go-http-client Disallow: / User-agent: trendiction Disallow: / User-agent: ELinks Disallow: / User-agent: proximic Disallow: / User-agent: serendeputy Disallow: / User-agent: PhantomJS Disallow: / User-agent: \*bot Disallow: / User-agent: Validator.nu Disallow: / User-agent: MojeekBot Disallow: / User-agent: W3C_Validator Disallow: / User-agent: Yahoo! Slurp Disallow: / User-agent: My Nutch Spider Disallow: / User-agent: survey Disallow: / User-agent: fetcher Disallow: / User-agent: nbot Disallow: / User-agent: DotBot Disallow: / User-agent: GrapeshotCrawler Disallow: / User-agent: crawl Disallow: / User-agent: CCBot Disallow: / User-agent: Links ( Disallow: / User-agent: msie8 - ( Rogue Robot ) Disallow: / User-agent: ZumBot Disallow: / User-agent: spider Disallow: / User-agent: MixrankBot Disallow: / User-agent: discover Disallow: / User-agent: scanner Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: ng/1 Disallow: / User-agent: UptimeRobot Disallow: / User-agent: daum Disallow: / User-agent: blog Disallow: / User-agent: XML Sitemaps Generator Disallow: / User-agent: scrapy Disallow: / User-agent: serendeputy Disallow: / User-agent: My Nutch Spider Disallow: / User-agent: A PHP script Disallow: / User-agent: Google Web Preview Disallow: / User-agent: SeznamBot Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: Yandex ( catchall ) Disallow: / User-agent: GetIntent Crawler Disallow: / User-agent: CUBOT Disallow: / User-agent: Baiduspider Disallow: / User-agent: survey Disallow: / User-agent: Curl Disallow: / User-agent: oBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: link Disallow: / User-agent: 2345Explorer Disallow: / User-agent: semanticbot Disallow: / User-agent: Wget Disallow: / User-agent: blog Disallow: / User-agent: Java ( catchall ) Disallow: / User-agent: feed Disallow: / User-agent: BLEXBot Disallow: / User-agent: Dalvik Disallow: / User-agent: link Disallow: / User-agent: Links ( Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: Konqueror Disallow: / User-agent: Feedfetcher-Google Disallow: / User-agent: YisouSpider Disallow: / User-agent: perl Disallow: / User-agent: checker Disallow: / User-agent: checker Disallow: / User-agent: Pinterestbot Disallow: / User-agent: MixrankBot Disallow: / User-agent: validator Disallow: / User-agent: semanticbot Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: YandexImageResizer Disallow: / User-agent: 360Spider Disallow: / User-agent: crawl Disallow: / User-agent: Sogou web spider Disallow: / User-agent: robot Disallow: / User-agent: SeznamBot Disallow: / User-agent: * Disallow: /?s= Disallow: /search User-agent: * Disallow: /trackback Disallow: /*trackback Disallow: /*trackback* Disallow: /*/trackback User-agent: * Allow: /feed/$ Disallow: /feed/ Disallow: /comments/feed/ Disallow: /*/feed/$ Disallow: /*/feed/rss/$ Disallow: /*/trackback/$ Disallow: /*/*/feed/$ Disallow: /*/*/feed/rss/$ Disallow: /*/*/trackback/$ Disallow: /*/*/*/feed/$ Disallow: /*/*/*/feed/rss/$ Disallow: /*/*/*/trackback/$ Sitemap: https://pivigames.blog/sitemap_index.xml Sitemap: https://pivigames.blog/post-sitemap.xml Sitemap: https://pivigames.blog/page-sitemap.xml Sitemap: https://pivigames.blog/category-sitemap.xml