# robots.txt for etrm.live
# Strategy: major search engines crawl freely (they drive traffic),
# everything else is throttled, and high-volume SEO scrapers with no
# referral value are blocked outright.
# Note: Crawl-delay is honored by Bing/Yandex and most well-behaved bots.
# Googlebot ignores it — Google crawl rate is managed in Search Console.
# Non-compliant bots are handled by rack-attack, not this file.

Sitemap: https://etrm.live/sitemaps/sitemap.xml.gz

# --- Major search engines: full access ---
User-agent: Googlebot
Disallow: /chat

User-agent: Bingbot
Crawl-delay: 5
Disallow: /chat

User-agent: DuckDuckBot
Crawl-delay: 5
Disallow: /chat

# --- Google's non-Search crawler (R&D/AI): no ranking impact, throttling
# --- crawl budget for Search-relevant bots. Compliant, so robots-blocked.
User-agent: GoogleOther
Disallow: /

# --- Heavy SEO/data scrapers: no referral value, blocked ---
User-agent: AionBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Reflectionbot
Disallow: /

User-agent: KeenableBot
Disallow: /

User-agent: LinkupBot
Disallow: /

User-agent: CriteoBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

# --- Everyone else: crawl slowly ---
User-agent: *
Crawl-delay: 10
Disallow: /chat