# RaceMob Bot Traffic Control - Updated for Hong Kong bot protection # See http://www.robotstxt.org/wc/norobots.html for documentation # Allow legitimate search engines with restrictions User-agent: Googlebot Disallow: /admin/ Disallow: /api/ Disallow: /signup Disallow: /sign_up Disallow: /registrations Disallow: /login Disallow: /suggested_races/ Disallow: /quickblog/ Disallow: /honeypot/ Disallow: /admin_fake/ Disallow: /race_results/ Disallow: /results/ Disallow: /content_recommendations/ Disallow: /races/*/results Disallow: /races/*/register_external Disallow: /races/*/participants Allow: / User-agent: Bingbot Disallow: /admin/ Disallow: /api/ Disallow: /signup Disallow: /sign_up Disallow: /registrations Disallow: /login Disallow: /suggested_races/ Disallow: /quickblog/ Disallow: /honeypot/ Disallow: /admin_fake/ Disallow: /race_results/ Disallow: /results/ Disallow: /content_recommendations/ Disallow: /races/*/results Disallow: /races/*/register_external Disallow: /races/*/participants Allow: / User-agent: Slurp Disallow: /admin/ Disallow: /suggested_races/ Disallow: /quickblog/ Disallow: /honeypot/ Disallow: /admin_fake/ Disallow: /api/ Disallow: /signup Disallow: /sign_up Disallow: /registrations Disallow: /login Disallow: /race_results/ Disallow: /results/ Disallow: /content_recommendations/ Disallow: /races/*/results Disallow: /races/*/register_external Disallow: /races/*/participants Allow: / # Block aggressive SEO crawlers and scrapers User-agent: AhrefsBot Disallow: / Disallow: /races/*/participants User-agent: SemrushBot Disallow: / Disallow: /races/*/participants User-agent: MJ12bot Disallow: / Disallow: /races/*/participants User-agent: DotBot Disallow: / Disallow: /races/*/participants User-agent: BLEXBot Disallow: / Disallow: /races/*/participants User-agent: MegaIndex Disallow: / Disallow: /races/*/participants User-agent: PetalBot Disallow: / Disallow: /races/*/participants User-agent: SiteAuditBot Disallow: / Disallow: /races/*/participants User-agent: LinkpadBot Disallow: / Disallow: /races/*/participants User-agent: YandexBot Disallow: / Disallow: /races/*/participants User-agent: BaiduSpider Disallow: / Disallow: /races/*/participants User-agent: SeznamBot Disallow: / Disallow: /races/*/participants User-agent: FacebookExternalHit Disallow: /admin/ Disallow: /suggested_races/ Disallow: /quickblog/ Disallow: /races/*/participants Allow: / User-agent: Twitterbot Disallow: /admin/ Disallow: /suggested_races/ Disallow: /quickblog/ Disallow: /races/*/participants Allow: / # Block all other bots (catch-all) User-agent: * Disallow: /admin/ Disallow: /api/ Disallow: /signup Disallow: /sign_up Disallow: /registrations Disallow: /login Disallow: /suggested_races/ Disallow: /quickblog/ Disallow: /honeypot/ Disallow: /admin_fake/ Disallow: /races/*/register_external Disallow: /race_results/ Disallow: /results/ Disallow: /content_recommendations/ Disallow: /races/*/results Disallow: /races/*/participants Crawl-delay: 2 # Sitemap location Sitemap: https://racemob.com/sitemap.xml.gz