# ════════════════════════════════════════════════════════════════
# robots.txt — webshturm.ru
# Generated at build time from src/lib/seo/bot-policy.ts
# Last build: 2026-09-27T02:52:32.160Z
# ════════════════════════════════════════════════════════════════

# ── Default ──
# All other crawlers — allow main content, deny service paths
User-agent: *
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# ── Search engines (primary traffic) ──
# Google Search
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Яндекс.Поиск
User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Bing Search
User-agent: bingbot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Mail.ru Поиск
User-agent: Mail.RU_Bot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# ── AI search agents (citation in answers) ──
# OpenAI ChatGPT browsing
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# OpenAI SearchGPT
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# ChatGPT plugin browsing
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Anthropic Claude (web search)
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Anthropic Claude (web fetching)
User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Perplexity AI search
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Perplexity user-initiated fetch
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Яндекс.Нейро (Яндекс AI search)
User-agent: YandexAdditional
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Google AI Overviews / SGE
User-agent: GoogleOther
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Bing Copilot AI fetch
User-agent: Bingbot-AI
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Cohere AI agent
User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# You.com search
User-agent: YouBot
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# ── AI training scrapers (hybrid policy) ──
# Common Crawl — generic training corpus
User-agent: CCBot
Disallow: /

# ByteDance/TikTok training (агрессивный)
User-agent: Bytespider
Disallow: /

# Amazon Alexa/Olympus training
User-agent: Amazonbot
Disallow: /

# Meta Llama training
User-agent: Meta-ExternalAgent
Disallow: /

# Meta general (FB заблокирован в РФ)
User-agent: FacebookBot
Disallow: /

# Hive image-AI training
User-agent: ImagesiftBot
Disallow: /

# Anthropic training scraper (consistency с ClaudeBot)
User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Google Gemini training (Google = main search)
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# Apple Intelligence training
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /go/
Disallow: /admin/
Disallow: /cookie-settings
Disallow: /subscription-confirmed
Disallow: /unsubscribed

# ── Competitor SEO scrapers (deny) ──
# Ahrefs SEO crawler
User-agent: AhrefsBot
Disallow: /

# SEMrush SEO crawler
User-agent: SemrushBot
Disallow: /

# Majestic SEO crawler
User-agent: MJ12bot
Disallow: /

# Moz / OpenSiteExplorer
User-agent: DotBot
Disallow: /

# WebMeUp scraping
User-agent: BLEXBot
Disallow: /

# Huawei Petal Search
User-agent: PetalBot
Disallow: /

# ── Yandex primary mirror ──
Host: webshturm.ru

# ── Sitemap ──
Sitemap: https://webshturm.ru/sitemap-index.xml