# ════════════════════════════════════════════════════════════════════════
# robots.txt — Brand Banao.Ai (https://brandbanao.ai/)
# ════════════════════════════════════════════════════════════════════════
#
# Strategy:
# • Allow all major search engine crawlers full access to the site.
# • Explicitly allow Google's AI/SGE bots (GPTBot, Google-Extended,
#   Anthropic, PerplexityBot) — critical for AIO / GEO ranking.
# • Block admin, API, and internal paths that should not be indexed.
# • Reference sitemap for fast discovery of all URLs.
#
# IMPORTANT: Place this file in your /public folder so it is served at:
#   https://brandbanao.ai/robots.txt
#
# ════════════════════════════════════════════════════════════════════════


# ── ALLOW: All standard search engine crawlers ───────────────────────
User-agent: *
Allow: /

# Block non-public paths (adjust to match your actual routes)
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /login/
Disallow: /register/
Disallow: /_next/          # if using Next.js
Disallow: /static/chunks/  # internal JS chunks (not content pages)

# Block low-value query strings (prevents crawl budget waste)
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?sessionid=

# Allow all assets needed for rendering (Google crawls CSS/JS)
Allow: /assets/
Allow: /images/
Allow: /fonts/
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.woff2$


# ── ALLOW: Google bots (including SGE / AI Overview) ─────────────────
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /assets/
Allow: /images/

User-agent: Googlebot-Video
Allow: /

# Google's AI training and SGE bot — allow for AIO ranking signals
User-agent: Google-Extended
Allow: /


# ── ALLOW: Bing / Microsoft ──────────────────────────────────────────
User-agent: Bingbot
Allow: /

User-agent: MSNBot
Allow: /


# ── ALLOW: AI Search Engines — critical for GEO / AEO / AIO ─────────
# Perplexity AI
User-agent: PerplexityBot
Allow: /

# OpenAI / ChatGPT search citations
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic Claude (web citations)
User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

# You.com AI search
User-agent: YouBot
Allow: /

# Meta AI (Llama-based search)
User-agent: FacebookBot
Allow: /

# Apple / Applebot (Siri search signals)
User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bingbot
Crawl-delay: 2

User-agent: YandexBot
Crawl-delay: 5


Sitemap: https://brandbanao.ai/sitemap.xml