# leonidbugaev.com - robots.txt
# Leonid Bugaev: business communications, networking and B2B sales expert
# Primary language: English (en). Russian (ru) canonicals live on https://lbugaev.com.
# Canonical site: https://leonidbugaev.com

# --- General crawlers ---

User-agent: *
Allow: /
Clean-param: b /

# --- Yandex crawler ---
# Russian canonicals live on https://lbugaev.com. Yandex does not follow
# cross-domain rel=canonical, so the /ru/ duplicates on this host are blocked
# explicitly to keep them out of the low-value pages report.
User-agent: Yandex
Disallow: /ru$
Disallow: /ru/
Clean-param: b&channel&from&utm_source&utm_medium&utm_campaign&utm_content&utm_term /

# --- AI and LLM crawlers (explicitly allowed) ---
# Structured text resources for AI systems:
#   https://leonidbugaev.com/ai.txt        - AI index (identity, expertise, services, contact)
#   https://leonidbugaev.com/llms.txt      - Canonical LLM summary
#   https://leonidbugaev.com/llms-full.txt - Extended context: services, blog topics, geography, languages

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Diffbot
Allow: /

# Agent description: https://leonidbugaev.com/.well-known/agent-description.md
Sitemap: https://leonidbugaev.com/sitemap-index.xml
Sitemap: https://leonidbugaev.com/sitemap-0.xml
Sitemap: https://leonidbugaev.com/en/rss.xml