# ==============================================================================
# ADVANCED AI-Optimized Robots.txt
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. ALLOW AI SEARCH & CITATION BOTS (Live Web Search & Generative Answers)
# ------------------------------------------------------------------------------
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: ChatGPT-User
User-agent: Bingbot
User-agent: Google-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /wp-json/
Disallow: /?s=
Disallow: /*?*

# ------------------------------------------------------------------------------
# 2. BLOCK AI DATA SCRAPERS (Model Training / Non-Attributed Content Scrapers)
# ------------------------------------------------------------------------------
User-agent: GPTBot
User-agent: CCBot
User-agent: Anthropic-AI
User-agent: ClaudeBot
User-agent: Bytespider
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: FacebookBot
User-agent: Amazonbot
Disallow: /

# ------------------------------------------------------------------------------
# 3. GLOBAL RULES & CRAWL BUDGET OPTIMIZATION (General Crawlers)
# ------------------------------------------------------------------------------
User-agent: *
Allow: /

# Block WP Core & Administrative Paths
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# Allow essential WP resource directories for rendering, block internal code
Allow: /wp-content/uploads/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-includes/
Allow: /wp-includes/js/
Allow: /wp-includes/css/
Allow: /wp-includes/images/

# Block Low-Value & Duplicate Content URLs
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /author/
Disallow: /page/*?*

# Block Query Strings & Internal Tracking Parameters (Prevents Infinite Crawl Loops)
Disallow: /*?*utm_*
Disallow: /*?*gclid=*
Disallow: /*?*fbclid=*

# ------------------------------------------------------------------------------
# 4. SITEMAP DECLARATIONS
# ------------------------------------------------------------------------------
Sitemap: https://www.masnad.com.au/sitemap_index.xml

# WPStaq Auto-Added Rules - Start
Crawl-delay: 10
# WPStaq Auto-Added Rules - End