# Open Labor Project - robots.txt
# Free automotive labor times, torque specs, fluid specs, DTC codes,
# battery specs, and repair procedures. 6M+ data points.

# ============================================================
# Default rules — apply to every crawler not listed below
# ============================================================
User-agent: *
Allow: /
Allow: /sitemaps/
Allow: /sitemap.xml
Disallow: /api/
Disallow: /admin/
Disallow: /account/
Disallow: /developer/
Disallow: /preview/
Allow: /_next/static/
Disallow: /_next/
# Tripwire: forbidden path. Polite crawlers obey; anything that fetches
# /api/honey/ has confessed it ignores robots and is scraping the link graph.
Disallow: /api/honey/
Crawl-delay: 2

# ============================================================
# Search engines — same rules as default, just custom crawl-delay
# ============================================================
User-agent: Googlebot
Allow: /
Allow: /sitemaps/
Allow: /sitemap.xml
Disallow: /api/
Disallow: /admin/
Disallow: /account/
Disallow: /developer/
Disallow: /preview/
Allow: /_next/static/
Disallow: /_next/
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Bing is our best-performing search channel (clicks doubled May-Aug 2026)
# and bingbot HONORS Crawl-delay, unlike Google — the old "Crawl-delay: 5"
# capped it at ~17k pages/day. No delay: the site serves in 100-300ms and
# Bing's crawler auto-tunes politely. Throttle in Bing Webmaster if ever
# needed, not here.
User-agent: bingbot
Allow: /
Allow: /sitemaps/
Allow: /sitemap.xml
Disallow: /api/
Disallow: /admin/
Disallow: /account/
Disallow: /developer/
Disallow: /preview/
Allow: /_next/static/
Disallow: /_next/

# ============================================================
# AI crawlers — welcome! Cite us, link us, train on us.
# Open data is the whole point. Please honor /api/ and /_next/.
# ============================================================

# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Google AI (separate from Googlebot — controls AI summaries / Gemini training)
User-agent: Google-Extended
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Apple Intelligence
User-agent: Applebot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Common Crawl (used to train many open-source AI models)
User-agent: CCBot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Meta AI
User-agent: meta-externalagent
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

User-agent: FacebookBot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# ByteDance / TikTok
User-agent: Bytespider
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Amazon Alexa / shopping
User-agent: Amazonbot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Mistral
User-agent: MistralAI-User
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# DeepSeek
User-agent: DeepSeekBot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# Diffbot (used by AI/RAG pipelines)
User-agent: Diffbot
Allow: /
Disallow: /api/
Allow: /_next/static/
Disallow: /_next/

# ============================================================
# Aggressive SEO crawlers — blocked entirely
# ============================================================
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# ============================================================
# Baidu — blocked 2026-09-20 (owner call)
# ============================================================
# Verified genuine Baidu (PTR resolves to *.crawl.baidu.com) and it obeyed
# every Disallow above: zero hits on /admin/, /account/, /api/. Blocked purely
# on economics. It pulled 424,119 pages in seven days, 321,354 of them
# distinct, and referred FOUR human visitors in sixty days. Roughly 100,000
# page renders per person sent.
#
# It also corrupted the human counter: Baiduspider-render executes JavaScript,
# so unlike Googlebot and bingbot it fired the first-party /api/echo beacon and
# was logged as ~250 "engaged humans" a day. That is fixed separately in
# pages/api/echo.js; this block removes the load as well.
#
# For scale, 30-day referrers: Google 2,424, DuckDuckGo 2,132, Bing 1,987,
# Yahoo 539, ChatGPT 213, Baidu 4.
User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-render
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-news
Disallow: /

# ============================================================
# Sitemap
# ============================================================
Sitemap: https://openlaborproject.com/sitemap.xml
# 2026-11-01: curated core back on. The Bing leaves index is submitted in Bing
# Webmaster Tools only, never listed here (Google must not get it).