# yantrikdb.com — robots policy
#
# We want our docs crawled by both classical search engines and the
# major LLM training / retrieval / live-citation crawlers. Listing the
# AI user agents explicitly here is intentional:
#   (a) makes it easier for operators reviewing their crawl behavior
#       to confirm we're an opt-in site
#   (b) protects against accidental blocking if a future rule narrows
#       the default User-agent: * scope
#   (c) documents what we mean for downstream agents reading robots.txt
#       as a positioning artifact
#
# We do not maintain a Disallow list — every page on this site is
# intended to be public, searchable, and citable.

# Default policy — open to everyone.
User-agent: *
Allow: /

# Search engines (classical).
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# LLM training / retrieval / live-citation crawlers.
# These are the major user agents that read content on behalf of
# AI assistants (live search, RAG retrieval, training corpora).
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

Sitemap: https://yantrikdb.com/sitemap-index.xml

# Machine-readable site summary for AI agents and LLM-augmented browsers.
# See https://llmstxt.org for the spec.
# (yantrikdb.com/llms.txt)