From 9267f11dbdbb1eb073865f06babfd0909837fe6f Mon Sep 17 00:00:00 2001 From: Eltrac Date: Sat, 17 May 2025 09:09:36 +0800 Subject: chore: add robots.txt --- static/robots.txt | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 static/robots.txt diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 00000000..cc5537d4 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,61 @@ +User-agent: Googlebot +Allow: / + +User-agent: Bingbot +Allow: / + +User-agent: DuckDuckBot +Allow: / + +User-agent: Baiduspider +Allow: / + +User-agent: Yandex +Allow: / + +# Allow link preview bots +User-agent: TelegramBot +Allow: / + +User-agent: Discordbot +Allow: / + +User-agent: Twitterbot +Allow: / + +User-agent: facebookexternalhit +Allow: / + +# Block known AI scrapers +User-agent: GPTBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: Amazonbot +Disallow: / + +User-agent: omgili +Disallow: / + +User-agent: Diffbot +Disallow: / + +# Disallow all others +User-agent: * +Disallow: / + +Sitemap: https://www.geedea.pro/sitemap.xml -- cgit v1.2.3