# =========================================================
# Welcome to Ustadio's robots.txt
# =========================================================

User-agent: *

# Disallow access to system/core/internal directories
Disallow: /admin/
Disallow: /api/
Disallow: /components/
Disallow: /config/
Disallow: /includes/
Disallow: /libs/
Disallow: /seo/
Disallow: /services/
Disallow: /views/

# ---------------------------------------------------------
# Sitemaps
# ---------------------------------------------------------
Sitemap: https://ustadio.com/sitemap.xml
Sitemap: https://ustadio.com/sitemap-static.xml
Sitemap: https://ustadio.com/sitemap-tutors.xml
Sitemap: https://ustadio.com/sitemap-cities.xml
Sitemap: https://ustadio.com/sitemap-subjects.xml
Sitemap: https://ustadio.com/sitemap-levels.xml
Sitemap: https://ustadio.com/sitemap-exams.xml

# Disallow specific scripts and sensitive files
Disallow: /setupdb
Disallow: /alterdb
Disallow: /setup_db.php
Disallow: /alter_db.php
Disallow: /*.sql$

# Prevent indexing of chaotic query parameters
Disallow: /*?sort=*
Disallow: /*?filter=*
Disallow: /*&sort=*
Disallow: /*&filter=*
Disallow: /*?q=*
Disallow: /*?search=*
Disallow: /*?category=*
Disallow: /*?slug=*

# ---------------------------------------------------------
# Search Engine Bots
# ---------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# ---------------------------------------------------------
# Ad Bots
# ---------------------------------------------------------
User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /
User-agent: AdsBot-Google-Mobile
Allow: /

# ---------------------------------------------------------
# IMAGE CRAWLERS - ALLOWED: OG, Twitter, Tutor Photos, Logos
# ---------------------------------------------------------

# GOOGLE IMAGE BOT
User-agent: Googlebot-Image

# FIRST: Block EVERYTHING in images folder
Disallow: /assets/images/

# THEN: Allow the specific allowed folders
Allow: /assets/images/og-images/
Allow: /assets/images/tutors/profile-photos/

# Allow specific logo files (even if they're in blocked folders)
Allow: /assets/images/ustadio_logo.png

# THEN: Block the default avatars WITHIN the allowed tutor folder
Disallow: /assets/images/tutors/profile-photos/male-avatar.png
Disallow: /assets/images/tutors/profile-photos/female-avatar.png

Disallow: /*.svg$
Disallow: /*.webp$

# BING IMAGE BOT
User-agent: Bingbot-Image

# Block EVERYTHING first
Disallow: /assets/images/

# Allow the specific allowed folders
Allow: /assets/images/og-images/
Allow: /assets/images/tutors/profile-photos/


# Allow specific logo files (even if they're in blocked folders)
Allow: /assets/images/ustadio_logo.png

# Block default avatars
Disallow: /assets/images/tutors/profile-photos/male-avatar.png
Disallow: /assets/images/tutors/profile-photos/female-avatar.png

# Block SVGs and WebPs
Disallow: /*.svg$
Disallow: /*.webp$

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

# ---------------------------------------------------------
# AI / LLM Scrapers (Block entirely)
# ---------------------------------------------------------
User-agent: CCBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Cohere-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

# ---------------------------------------------------------
# SEO Audit Scraping Bots (Aggressive)
# ---------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /