# robots.txt for scaledojo.dev - System Design Practice Platform

# Default rules for all crawlers
User-agent: *
Allow: /
# Public content
Allow: /wiki
Allow: /about
Allow: /blogs
Allow: /papers
Allow: /pricing
Allow: /roadmap
Allow: /faq
Allow: /cheatsheet
Allow: /challenges
Allow: /be-an-affiliate
Allow: /llms.txt
Allow: /llms-full.txt
# Lab hub pages (public overview pages)
Allow: /lab
Allow: /lld
Allow: /sql
Allow: /api-design
Allow: /genai
Allow: /forge
# Origin story pages (public)
Allow: /hld/origins
Allow: /lld/origins
Allow: /genai/origins
Allow: /api-design/origins
Allow: /forge/origins
# Private: auth-required pages
Disallow: /dashboard
Disallow: /profile
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth
Disallow: /reset-password
Disallow: /verify-email
Disallow: /progress
Disallow: /influencer
Disallow: /marketing/
# Referral tracking links - redirect-only, no unique content of their own
Disallow: /r/
# Private: individual lab levels (require login; redirect to auth for crawlers)
# Custom Sandbox is the one exception - free to use with no signup, so it stays
# crawlable even though every other /playground/ route (saved/shared designs)
# requires login and shouldn't be indexed.
Allow: /playground/custom$
Disallow: /playground/
Disallow: /lld/lab/
Disallow: /api-design/lab/
Disallow: /genai/lab/
Disallow: /forge/lab/
# Private: arena, peer, avatar, leaderboard (ProtectedRoute)
Disallow: /arena
Disallow: /leaderboard
Disallow: /peer/
Disallow: /avatar/
# Misc private
Disallow: /policy/

# Sitemap location
Sitemap: https://scaledojo.dev/sitemap.xml

# Google
User-agent: Googlebot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth
Disallow: /progress
Disallow: /influencer
Disallow: /marketing/
Disallow: /r/
Allow: /playground/custom$
Disallow: /playground/
Disallow: /lld/lab/
Disallow: /api-design/lab/
Disallow: /genai/lab/
Disallow: /forge/lab/
Disallow: /arena
Disallow: /leaderboard
Disallow: /peer/
Disallow: /avatar/
Disallow: /policy/

# Bing
User-agent: Bingbot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth
Disallow: /progress
Disallow: /influencer
Disallow: /marketing/
Disallow: /r/
Allow: /playground/custom$
Disallow: /playground/
Disallow: /lld/lab/
Disallow: /api-design/lab/
Disallow: /genai/lab/
Disallow: /forge/lab/
Disallow: /arena
Disallow: /leaderboard
Disallow: /peer/
Disallow: /avatar/
Disallow: /policy/

# ChatGPT / OpenAI crawler
User-agent: GPTBot
Allow: /wiki
Allow: /about
Allow: /blogs
Allow: /papers
Allow: /lab
Allow: /lld
Allow: /sql
Allow: /api-design
Allow: /genai
Allow: /forge
Allow: /hld/learn
Allow: /lld/learn
Allow: /api-design/learn
Allow: /genai/learn
Allow: /forge/learn
Allow: /cheatsheet
Allow: /challenges
Allow: /roadmap
Allow: /pricing
Allow: /faq
Allow: /changelog
Allow: /be-an-affiliate
Allow: /hld/origins
Allow: /lld/origins
Allow: /genai/origins
Allow: /api-design/origins
Allow: /forge/origins
Allow: /playground/custom$
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /

# Claude / Anthropic crawler
User-agent: ClaudeBot
Allow: /wiki
Allow: /about
Allow: /blogs
Allow: /papers
Allow: /lab
Allow: /lld
Allow: /sql
Allow: /api-design
Allow: /genai
Allow: /forge
Allow: /hld/learn
Allow: /lld/learn
Allow: /api-design/learn
Allow: /genai/learn
Allow: /forge/learn
Allow: /cheatsheet
Allow: /challenges
Allow: /roadmap
Allow: /pricing
Allow: /faq
Allow: /changelog
Allow: /be-an-affiliate
Allow: /hld/origins
Allow: /lld/origins
Allow: /genai/origins
Allow: /api-design/origins
Allow: /forge/origins
Allow: /playground/custom$
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /

# Google AI / Gemini
User-agent: Google-Extended
Allow: /wiki
Allow: /about
Allow: /blogs
Allow: /papers
Allow: /lab
Allow: /lld
Allow: /sql
Allow: /api-design
Allow: /genai
Allow: /forge
Allow: /hld/learn
Allow: /lld/learn
Allow: /api-design/learn
Allow: /genai/learn
Allow: /forge/learn
Allow: /cheatsheet
Allow: /challenges
Allow: /roadmap
Allow: /pricing
Allow: /faq
Allow: /changelog
Allow: /be-an-affiliate
Allow: /hld/origins
Allow: /lld/origins
Allow: /genai/origins
Allow: /api-design/origins
Allow: /forge/origins
Allow: /playground/custom$
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /

# Perplexity AI crawler
User-agent: PerplexityBot
Allow: /wiki
Allow: /about
Allow: /blogs
Allow: /papers
Allow: /lab
Allow: /lld
Allow: /sql
Allow: /api-design
Allow: /genai
Allow: /forge
Allow: /hld/learn
Allow: /lld/learn
Allow: /api-design/learn
Allow: /genai/learn
Allow: /forge/learn
Allow: /cheatsheet
Allow: /challenges
Allow: /roadmap
Allow: /pricing
Allow: /faq
Allow: /changelog
Allow: /be-an-affiliate
Allow: /hld/origins
Allow: /lld/origins
Allow: /genai/origins
Allow: /api-design/origins
Allow: /forge/origins
Allow: /playground/custom$
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /

# OpenAI ChatGPT search / browsing (distinct from GPTBot training crawler)
User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth
Disallow: /progress
Disallow: /influencer
Disallow: /marketing/
Disallow: /r/
Allow: /playground/custom$
Disallow: /playground/
Disallow: /lld/lab/
Disallow: /api-design/lab/
Disallow: /genai/lab/
Disallow: /forge/lab/
Disallow: /arena
Disallow: /leaderboard
Disallow: /peer/
Disallow: /avatar/
Disallow: /policy/

# OpenAI search indexing bot
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth
Disallow: /progress
Disallow: /influencer
Disallow: /marketing/
Disallow: /r/
Allow: /playground/custom$
Disallow: /playground/
Disallow: /lld/lab/
Disallow: /api-design/lab/
Disallow: /genai/lab/
Disallow: /forge/lab/
Disallow: /arena
Disallow: /leaderboard
Disallow: /peer/
Disallow: /avatar/
Disallow: /policy/

# Claude browsing / search (distinct from ClaudeBot training crawler)
User-agent: Claude-User
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth
Disallow: /progress
Disallow: /influencer
Disallow: /marketing/
Disallow: /r/
Allow: /playground/custom$
Disallow: /playground/
Disallow: /lld/lab/
Disallow: /api-design/lab/
Disallow: /genai/lab/
Disallow: /forge/lab/
Disallow: /arena
Disallow: /leaderboard
Disallow: /peer/
Disallow: /avatar/
Disallow: /policy/

User-agent: Claude-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth
Disallow: /progress
Disallow: /influencer
Disallow: /marketing/
Disallow: /r/
Allow: /playground/custom$
Disallow: /playground/
Disallow: /lld/lab/
Disallow: /api-design/lab/
Disallow: /genai/lab/
Disallow: /forge/lab/
Disallow: /arena
Disallow: /leaderboard
Disallow: /peer/
Disallow: /avatar/
Disallow: /policy/

# Perplexity user browsing (distinct from PerplexityBot training crawler)
User-agent: Perplexity-User
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /auth
Disallow: /progress
Disallow: /influencer
Disallow: /marketing/
Disallow: /r/
Allow: /playground/custom$
Disallow: /playground/
Disallow: /lld/lab/
Disallow: /api-design/lab/
Disallow: /genai/lab/
Disallow: /forge/lab/
Disallow: /arena
Disallow: /leaderboard
Disallow: /peer/
Disallow: /avatar/
Disallow: /policy/

# Meta AI crawler
User-agent: FacebookBot
Allow: /

# Apple / Siri
User-agent: Applebot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /