From 7e2566b49321a89c49f8644c031245aa233bc937 Mon Sep 17 00:00:00 2001
From: fynks <75840152+fynks@users.noreply.github.com>
Date: Mon, 27 Oct 2025 10:37:44 +0500
Subject: [PATCH] fix(robots.txt): remove disallow rules for cgi-bin, admin,
 and JSON files

---
 dist/robots.txt | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/dist/robots.txt b/dist/robots.txt
index 1cf3f23..8263d98 100644
--- a/dist/robots.txt
+++ b/dist/robots.txt
@@ -1,9 +1,6 @@
 # robots.txt for debridcompare.xyz
 User-agent: *
 Allow: /
-Disallow: /cgi-bin/
-Disallow: /admin/
-Disallow: /*.json$
 
 # Crawl-delay for respectful crawling
 Crawl-delay: 1
@@ -14,11 +11,11 @@ Sitemap: https://debridcompare.xyz/sitemap.xml
 # Specific user agents
 User-agent: Googlebot
 Allow: /
-Crawl-delay: 0
+Crawl-delay: 1
 
 User-agent: Bingbot
 Allow: /
-Crawl-delay: 0
+Crawl-delay: 1
 
 User-agent: Slurp
 Allow: /
@@ -28,4 +25,4 @@ Allow: /
 
 User-agent: Baiduspider
 Allow: /
-Crawl-delay: 2
+Crawl-delay: 1