Sitemap: https://www.titrespresse.com/sitemap-news-index.xml

# ---------------------------------
# GLOBAL RULES FOR ALL BOTS (Googlebot, Bingbot, etc.)
# ---------------------------------
User-agent: *
Disallow: /results/
Disallow: /us/

# ---------------------------------
# COMPLETELY BLOCKED BOTS (AI, Scrapers, Aggressive Crawlers)
# ---------------------------------
User-agent: ClaudeBot
User-agent: PetalBot
User-agent: grapeshot
User-agent: Amazonbot
User-agent: Amzn-SearchBot
Disallow: /

# ---------------------------------
# CRAWL DELAYS FOR HEAVY SOCIAL/APP BOTS
# ---------------------------------
User-agent: facebookexternalhit
Crawl-delay: 25

User-agent: meta-externalagent
Crawl-delay: 25

User-agent: Timpibot
Crawl-delay: 10

User-agent: applebot
Crawl-delay: 5