# # robots.txt for TreeGraph # # http://help.yandex.ru/webmaster/?id=996567 # http://www.mediawiki.org/wiki/Robots.txt/ru # User-agent: * Disallow: /*query_id= Disallow: /internal/ Clean-param: template / Clean-param: query_id / Crawl-delay: 20 User-agent: AhrefsBot Disallow: / User-agent: GPTBot Disallow: / # Host: www.*** # Sitemap: http://www.***/sitemap.xml