# BEGIN Magic robots.txt # --------------------------- # General User-agent: * Allow: /phs/ Allow: /phs/wp-admin/admin-ajax.php Disallow: /phs/wp-admin/ Disallow: /phs/readme.html$ # Ad networks User-agent: Mediapartners-Google Disallow: /phs/ User-agent: AdsBot-Google Disallow: /phs/ User-agent: proximic Disallow: /phs/ User-agent: CriteoBot/0.1 Disallow: /phs/ User-agent: grapeshot Disallow: /phs/ # Link analyzers User-agent: AhrefsBot Disallow: /phs/ User-agent: SemrushBot Disallow: /phs/ User-agent: MJ12Bot Disallow: /phs/ User-agent: dotbot Disallow: /phs/ User-agent: BLEXBot Disallow: /phs/ User-agent: serpstatbot Disallow: /phs/ User-agent: DataForSeoBot Disallow: /phs/ User-agent: omgilibot Disallow: /phs/ User-agent: barkrowler Disallow: /phs/ # Downloaders User-agent: webreaper Disallow: /phs/ User-agent: MSIECrawler Disallow: /phs/ User-agent: WebCopier Disallow: /phs/ User-agent: HTTrack Disallow: /phs/ User-agent: VelenPublicWebCrawler Disallow: /phs/ User-agent: AwarioBot Disallow: /phs/ User-agent: AwarioRssBot Disallow: /phs/ User-agent: AwarioSmartBot Disallow: /phs/ # --------------------------- # END Magic robots.txt