# =====================================
# 🤖 Robots.txt for https://www.jxp.com
# =====================================

# -------------------------------------
# 🌐 General rules for all crawlers
# -------------------------------------
User-agent: *
Allow: /
Allow: /ai-lip-sync/infinitetalk-ai
Allow: /sora/sora2
Allow: /sora
Allow: /veo
Allow: /veo/veo-3
Allow: /chronoedit
Disallow: /api/        # API endpoints (not meant for indexing)
Disallow: /admin/      # Admin dashboard (protected)
Disallow: /dashboard/  # User dashboard (protected)
Disallow: /profile/    # User profiles (protected)
Disallow: /custom-pricing/  # Custom pricing page (not for indexing)
Disallow: /search?*    # Search result pages (avoid duplicate content)
Disallow: /*?*         # URLs with parameters (avoid duplicate content)
# Note: Static assets are allowed (no Disallow for /static/)

# -------------------------------------
# 🚫 External domain restrictions
# -------------------------------------
# Note: robots.txt only applies to the current domain
# External domains need their own robots.txt files

# -------------------------------------
# 🔍 Googlebot rules
# -------------------------------------
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /custom-pricing/

# -------------------------------------
# 🔎 Bingbot rules (search indexing allowed)
# -------------------------------------
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /custom-pricing/

# -------------------------------------
# 🔎 Baiduspider rules
# -------------------------------------
User-agent: Baiduspider
Allow: /
Disallow: /api/
Disallow: /custom-pricing/

# -------------------------------------
# 🤖 AI Crawler Rules (LLM-specific bots - restricted)
# -------------------------------------
# Training data collection bots (restricted)
User-agent: GPTBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Bytespider
User-agent: Diffbot
Allow: /llms.txt
Disallow: /

# AI search/agent bots (allowed for search but not training)
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /custom-pricing/

# -------------------------------------
# 📌 Sitemap
# -------------------------------------
Sitemap: https://www.jxp.com/sitemap.xml