# VaultSpark Studios — robots.txt

# Standard crawlers — welcome
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# ---------------------------------------------------------------
# AI training crawlers — disallowed
# Content on this site is proprietary to VaultSpark Studios LLC.
# Scraping for AI training is not permitted.
# ---------------------------------------------------------------

# OpenAI
User-agent: GPTBot
Disallow: /

# OpenAI search + explicit user-request retrieval are allowed to read the
# public site, but retain the same private-route exclusions as ordinary
# crawlers. GPTBot above remains the separate training opt-out.
User-agent: ChatGPT-User
Allow: /
Disallow: /vault-member/
Disallow: /investor-portal/
Disallow: /investor/
Disallow: /studio-hub/
Disallow: /.claude/
Allow: /.well-known/llms.txt
Allow: /.well-known/llms-full.txt
Allow: /.well-known/entity-graph.json
Allow: /.well-known/security.txt
Disallow: /.well-known/

User-agent: OAI-SearchBot
Allow: /
Disallow: /vault-member/
Disallow: /investor-portal/
Disallow: /investor/
Disallow: /studio-hub/
Disallow: /.claude/
Allow: /.well-known/llms.txt
Allow: /.well-known/llms-full.txt
Allow: /.well-known/entity-graph.json
Allow: /.well-known/security.txt
Disallow: /.well-known/

# Claude and Perplexity search + user-requested retrieval (D-S368.4): allowed
# so answers can cite the studio. Their training crawlers stay disallowed below.
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Perplexity-User
Allow: /
Disallow: /vault-member/
Disallow: /investor-portal/
Disallow: /investor/
Disallow: /studio-hub/
Disallow: /.claude/
Allow: /.well-known/llms.txt
Allow: /.well-known/llms-full.txt
Allow: /.well-known/entity-graph.json
Allow: /.well-known/security.txt
Disallow: /.well-known/

# Anthropic
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Google AI (Gemini / Bard training)
User-agent: Google-Extended
Disallow: /

# Meta
User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /

# Common Crawl (feeds many AI training pipelines)
User-agent: CCBot
Disallow: /

# Apple AI
User-agent: Applebot-Extended
Disallow: /

# Amazon Alexa / Diffbot
User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

# DataForSEO (scraper-as-a-service)
User-agent: DataForSeoBot
Disallow: /

# ---------------------------------------------------------------
# All other crawlers — public pages allowed, private areas blocked
# ---------------------------------------------------------------

User-agent: *
Allow: /

# Auth + private portals — no indexing
Disallow: /vault-member/
Disallow: /vault-member/admin/
Disallow: /investor-portal/
Disallow: /investor/
Disallow: /studio-hub/
Disallow: /.claude/

# AI-discovery corpus (CANON-048) — these four files are the PUBLIC citable
# surface advertised by agents.json + llms.txt. Longest-match Allow beats the
# directory Disallow below, so compliant crawlers can fetch exactly these
# while the rest of /.well-known/ stays private.
Allow: /.well-known/llms.txt
Allow: /.well-known/llms-full.txt
Allow: /.well-known/entity-graph.json
Allow: /.well-known/security.txt
Disallow: /.well-known/

# Sitemap
Sitemap: https://vaultsparkstudios.com/sitemap.xml

# AI-agent discovery manifest (CANON-011) — canonical machine-readable surfaces,
# primary CTA, policies, and citable per-project llms-full.txt shards.
# Agents: https://vaultsparkstudios.com/agents.json

# ---------------------------------------------------------------
# Note: the live robots.txt served at vaultsparkstudios.com may
# include additional directives injected by Cloudflare AI Labyrinth
# at the CDN edge. Those directives are not part of this file and
# are not committed to the repo. The source of truth for crawl rules
# controlled by VaultSpark Studios is the content above.
# ---------------------------------------------------------------