# Smart Picks Guide - robots.txt
# Updated: 2025-11-29
# Mode: Hybrid Smart (SEO + LLMO + bad-bot protection)

# -------- Global defaults (apply to all bots without specific rules) --------
User-agent: *
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

# -------- Major search engines --------
User-agent: Googlebot
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: Bingbot
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: DuckDuckBot
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: Applebot
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

# -------- AI / LLM-connected crawlers (kept ON for LLMO) --------
User-agent: GPTBot               # OpenAI crawler
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: ChatGPT-User         # ChatGPT browsing fetcher
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: Google-Extended      # Google AI data access
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: CCBot                # Common Crawl (feeds many LLMs)
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: PerplexityBot        # Perplexity AI search/Q&A
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: ClaudeBot            # Anthropic crawler
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: anthropic-ai         # Additional Anthropic user-agent
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: YouBot               # You.com AI search
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

# -------- Social link preview bots --------
User-agent: FacebookExternalHit  # Meta / Facebook / Instagram
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

User-agent: Twitterbot           # X (Twitter) previews
Allow: /
Disallow: /assets/styles.css 
Disallow: /cgi-bin/
Disallow: /qj874KFErq/
Disallow: /mail.smartpicksguide.com/
Disallow: /assets/
Disallow: /curator/
Disallow: /snippets/
Disallow: /wp-old/
Disallow: /pages/
Disallow: /articles/
Disallow: /no-bots-here/

# -------- Politely block noisy SEO / marketing crawlers --------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

# -------- Sitemap --------
Sitemap: https://www.smartpicksguide.com/sitemap.xml