# robots.txt for tools.newzone.top — 2026 AI-aware configuration
# Goal: maximize AI citations + traditional search visibility, block pure
# training scrapes. Reference:
# ~/.claude/skills/seo-geo/references/ai-bot-config.md
#
# Update policy: edit this file directly. next-sitemap is configured with
# generateRobotsTxt: false so it won't overwrite this.

# 1) Default — allow all, no sensitive paths on this site
User-agent: *
Allow: /

# 2) Traditional search engines — fully allowed (SEO foundation)
User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: 360Spider
Allow: /

User-agent: Sogou web spider
Allow: /

# 3) OpenAI — allow search + user fetches, block training
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-AdsBot
Allow: /

User-agent: GPTBot
Disallow: /

# 4) Anthropic / Claude — allow search + user fetches, block training
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Disallow: /

# 5) Perplexity — full allow (both bots drive citations)
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# 6) Chinese AI engines — allow (Bytespider also serves 豆包/头条/抖音,
#    blocking it would cost Chinese AI visibility)
User-agent: Bytespider
Allow: /

User-agent: Doubaobot
Allow: /

User-agent: Kimi-SearchBot
Allow: /

User-agent: Kimi-User
Allow: /

User-agent: KimiBot
Disallow: /

# 7) Google / Apple AI training opt-out
#    (Keep Googlebot / Applebot open for search; opt out of Gemini /
#    Apple Intelligence training only)
User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# 8) Other pure-training crawlers — block
User-agent: CCBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: cohere-ai
Disallow: /

# 9) Live retrieval / user-action bots — allow
User-agent: Meta-ExternalFetcher
Allow: /

# 10) Social preview bots — allow
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# 11) Cloudflare Content Signals (search OK, AI input OK, training NOT OK)
policy: content-use: search=yes; ai-input=yes; ai-train=no

# 12) Sitemap
Host: https://tools.newzone.top
Sitemap: https://tools.newzone.top/sitemap.xml