danvics/debrid-services-comparison
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(robots.txt): remove unnecessary user-agent rules and crawl-delay settings

Browse source 
fix(sitemap.xml): update homepage URL and change frequency; add flowchart image with caption
This commit is contained in:
fynks 2025-11-13 09:29:12 +05:00
parent 89c6258840
commit b6232cfc8c
2 changed files with 10 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
dist/robots.txt vendored
View file

@ -2,27 +2,5 @@
User-agent: *
Allow: /
# Crawl-delay for respectful crawling
Crawl-delay: 1
# Sitemaps
Sitemap: https://debridcompare.xyz/sitemap.xml
# Specific user agents
User-agent: Googlebot
Allow: /
Crawl-delay: 1
User-agent: Bingbot
Allow: /
Crawl-delay: 1
User-agent: Slurp
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Baiduspider
Allow: /
Crawl-delay: 1

13
dist/sitemap.xml vendored
View file

@ -7,13 +7,20 @@
<!-- Homepage -->
<url>
<loc>https://debridcompare.xyz/</loc>
<lastmod>2025-01-11T00:00:00+00:00</lastmod>
<changefreq>weekly</changefreq>
<loc>https://debridcompare.xyz</loc>
<lastmod>2025-11-13T00:00:00+00:00</lastmod>
<changefreq>daily</changefreq>
<priority>1.0</priority>
<image:image>
<image:loc>https://debridcompare.xyz/images/og.png</image:loc>
<image:title>Debrid Services Comparison Tool</image:title>
<image:caption>Compare 10 premium debrid services including AllDebrid, Real-Debrid, TorBox, and more</image:caption>
</image:image>
<image:image>
<image:loc>https://debridcompare.xyz/images/flowchart.svg</image:loc>
<image:title>How Debrid Services Work Flowchart</image:title>
<image:caption>A visual explanation of how debrid services provide high-speed downloads.</image:caption>
</image:image>
</url>
</urlset>