# This file is the single source of truth (Cloudflare "managed robots.txt" is OFF since
# 2026-08-14 — it used to prepend AI-bot blocks incl. GPTBot/ClaudeBot, killing AI-search
# citations). AI-SEARCH crawlers (GPTBot, ClaudeBot, PerplexityBot, OAI-SearchBot…) are
# deliberately ALLOWED; AI-TRAINING-only and bulk-scraper bots are blocked below.

User-agent: *
Allow: /

# Disallow private routes
Disallow: /dashboard/
Disallow: /account/
Disallow: /api/

# Affiliate redirect hops — crawling them just bounces bots to Shopee
Disallow: /r/
Disallow: /s/

# AI-TRAINING-only opt-outs (search/citation unaffected) + bulk scrapers
User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

# Block SEO / backlink crawlers (competitor intelligence — Ahrefs, Semrush, Majestic, Moz, …)
User-agent: AhrefsBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Sitemap
Sitemap: https://hoantien365.com/sitemap.xml