Sitemap: https://www.billtrack50.com/sitemap-index.xml
Crawl-Delay: 10

# Allow major search engines and social platforms
User-Agent: Slurp
Crawl-Delay: 10
Allow: /
Disallow: /Account/
Disallow: /Admin/
Disallow: /Error/
Disallow: /Public/

User-Agent: bingbot
Crawl-Delay: 10
Allow: /
Disallow: /Account/
Disallow: /Admin/
Disallow: /Error/
Disallow: /Public/

User-agent: Googlebot
Crawl-Delay: 10
Allow: /
Disallow: /Account/
Disallow: /Admin/
Disallow: /Error/
Disallow: /Public/

User-agent: Twitterbot
Crawl-Delay: 10
Allow: /
Disallow: /Account/
Disallow: /Admin/
Disallow: /Error/
Disallow: /Public/

User-agent: GPTBot
Disallow: /

#User-agent: ClaudeBot
#Disallow: /

# Allow legitimate research tools and web fetchers
User-agent: research
User-agent: academic
Disallow: /Account/
Disallow: /Admin/
Disallow: /Error/
Disallow: /Public/

# Block known problematic bots
User-agent: BLEXBot
Disallow: /

User-agent: facebookexternalhit
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Barkrowler
Disallow: /

# Block aggressive scrapers and commercial crawlers
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: 008
User-agent: TerraCotta
User-agent: Sogou web spider
Disallow: /

# Default policy for other bots - allow with restrictions
User-agent: *
Crawl-delay: 10
Disallow: /Account/
Disallow: /Admin/
Disallow: /Error/
Disallow: /Public/
Disallow: /api/
Disallow: /*?
Disallow: /search?
Disallow: /login
Disallow: /register