# ===============================
# General Access for Good Bots
# ===============================
User-agent: *
#Disallow: /admin
#Disallow: /admin/
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /private/
Disallow: /tmp/
Allow: /wp-admin/admin-ajax.php

# ===============================
# Disallow AI Crawlers & Scrapers
# ===============================

# OpenAI GPTBot
User-agent: GPTBot
Disallow: /

# ClaudeBot (Anthropic)
User-agent: ClaudeBot
Disallow: /

# CCBot (Common Crawl)
User-agent: CCBot
Disallow: /

# Amazonbot (AI scraping)
User-agent: Amazonbot
Disallow: /

# Sogou (China-based crawler, often ignored robots.txt)
User-agent: Sogou web spider
Disallow: /

# Ahrefs (SEO crawler, bandwidth-heavy)
User-agent: AhrefsBot
Disallow: /

# Semrush (same as above)
User-agent: SemrushBot
Disallow: /

# MJ12bot (link indexing, not useful for most)
User-agent: MJ12bot
Disallow: /