# robots.txt for Vexus Studios - Hardened Configuration
# Last updated: 2025-12-23

# === GOOD SEARCH ENGINES ===
User-agent: Googlebot
Allow: /
Disallow: /api/

User-agent: Googlebot-Image
Allow: /assets/
Disallow: /api/

User-agent: Bingbot
Allow: /
Disallow: /api/

User-agent: Slurp
Allow: /
Disallow: /api/

User-agent: DuckDuckBot
Allow: /
Disallow: /api/

User-agent: Baiduspider
Allow: /
Disallow: /api/

User-agent: YandexBot
Allow: /
Disallow: /api/

# === AI CRAWLERS - BLOCKED ===
# OpenAI GPT
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

# Anthropic Claude
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Google AI
User-agent: Google-Extended
Disallow: /

User-agent: GoogleOther
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

# Omgili
User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

# Facebook/Meta AI
User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Applebot-Extended (Apple AI)
User-agent: Applebot-Extended
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# Bytespider (ByteDance/TikTok AI)
User-agent: Bytespider
Disallow: /

# PerplexityBot
User-agent: PerplexityBot
Disallow: /

# YouBot (You.com)
User-agent: YouBot
Disallow: /

# ClaudeBot
User-agent: ClaudeBot
Disallow: /

# === SCRAPING/DATA HARVESTING BOTS - BLOCKED ===
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Seekport
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: spbot
Disallow: /

User-agent: BrandVerity
Disallow: /

User-agent: LinkpadBot
Disallow: /

User-agent: Cliqzbot
Disallow: /

# === AGGRESSIVE SCRAPERS - BLOCKED ===
User-agent: HTTrack
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: Wget
Disallow: /

User-agent: curl
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: scrapy
Disallow: /

# === GENERAL RULES FOR REMAINING BOTS ===
User-agent: *
Allow: /
Disallow: /api/
Crawl-delay: 2

# === SITEMAP ===
Sitemap: https://vexusstudios.com/sitemap.xml