# ============================================================
# Betfun24 robots.txt — Web & AI Crawlers
# Ultra-IA Structured Version
# ============================================================
# Default rules for all standard web crawlers
User-agent: *
Disallow: /admin/
Disallow: /internal/
Disallow: /private/
Allow: /
# Sitemaps (standard + AI-focused)
Sitemap: https://www.betfun24.com/sitemap.xml
Sitemap: https://www.betfun24.com/ai-sitemap.xml
# ------------------------------------------------------------
# AI / LLM CRAWLERS
# These sections are indicative and may be updated over time.
# Always refer to each provider's documentation for exact names.
# ------------------------------------------------------------
# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /private/
# Google AI / Google-Extended
User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /private/
# Anthropic Claude (example pattern)
User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /private/
# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /private/

# Other generic AI bots
User-agent: *AI*
Allow: /
Disallow: /admin/
Disallow: /internal/
Disallow: /private/
# ------------------------------------------------------------
# LLM GUIDELINES REFERENCE
# ------------------------------------------------------------
# LLMs and AI crawlers should read:
# - https://www.betfun24.com/llms.txt (global AI framework)
# - https://www.betfun24.com/ai-sitemap.xml (AI-optimised sitemap)
# as the primary reference for summarisation, brand identity
# and responsible gambling policy.
# End of robots.txt