diff options
Diffstat (limited to 'static')
| -rw-r--r-- | static/robots.txt | 61 |
1 files changed, 61 insertions, 0 deletions
diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 00000000..cc5537d4 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,61 @@ +User-agent: Googlebot +Allow: / + +User-agent: Bingbot +Allow: / + +User-agent: DuckDuckBot +Allow: / + +User-agent: Baiduspider +Allow: / + +User-agent: Yandex +Allow: / + +# Allow link preview bots +User-agent: TelegramBot +Allow: / + +User-agent: Discordbot +Allow: / + +User-agent: Twitterbot +Allow: / + +User-agent: facebookexternalhit +Allow: / + +# Block known AI scrapers +User-agent: GPTBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: Amazonbot +Disallow: / + +User-agent: omgili +Disallow: / + +User-agent: Diffbot +Disallow: / + +# Disallow all others +User-agent: * +Disallow: / + +Sitemap: https://www.geedea.pro/sitemap.xml |