TruckCodex robots policy
#
# Public FMCSA / NHTSA reference content is open to traditional search crawlers
# (Googlebot, Bingbot) and major AI retrieval bots (rate-limited at Cloudflare).
# SEO / competitive-intel scrapers remain Disallowed.
# See https://truckcodex.com/llms.txt for attribution preference.
#
# Content signals (https://contentsignals.org/):
#   search    = building a search index + returning hyperlinks + short excerpts
#   ai-input  = inputting content into AI models (RAG / grounding / live answers)
#   ai-train  = training or fine-tuning AI models
#
# Posture (2026-09-17): search=yes; ai-input=yes; ai-train=no.
# Googlebot / Bingbot / Google-InspectionTool remain fully unrestricted.
# Edge: express/scripts/setup-cloudflare-bot-throttling.mjs
#   ΓÇö Google/Bing skip; AI allowed + ~10/min heavy-path RL; SEO blocked.

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
# /api/ ΓÇö JSON endpoints consumed only by client JS (suggest, vin-recalls).
Disallow: /api/
# /compare/ ΓÇö hub + combinatorial pairs; also noindex,follow in HTML.
Disallow: /compare/

# ΓöÇΓöÇ Major AI retrieval bots ΓÇö ALLOW (CF rate-limits origin pressure) ΓöÇΓöÇ
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: AI2Bot
Allow: /

# ΓöÇΓöÇ Bulk training scrapers (no citation value) ΓöÇΓöÇ
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

# ΓöÇΓöÇ SEO / competitive-intel scrapers ΓöÇΓöÇ
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

# Sitemap ΓÇö flat root index
Sitemap: https://truckcodex.com/sitemap.xml