# Non-indexable paths (auth-gated, API, demo data).
# Shared Disallow list applied to every user-agent group below because
# named-UA groups don't inherit rules from `User-agent: *`.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

# AI crawlers — explicitly allowed for public pages, same private paths blocked.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

User-agent: Baiduspider
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

# Bing — explicit to avoid any ambiguity with the * group
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /history
Disallow: /history/
Disallow: /remix
Disallow: /sample/

Sitemap: https://toktranscript.com/sitemap.xml

# Content usage preferences (https://contentsignals.org/)
Content-Signal: ai-train=no, search=yes, ai-input=yes