# robots.txt for andrew.ooo
# Welcome, robots and AI agents! Everything here is crawlable —
# for search, for AI grounding, and for training.
# The one exception: /analytics/ is the self-hosted Rybbit ingest/proxy
# (script.js, track, session-replay) — pure crawl-budget noise, not content.

User-agent: *
Allow: /
Disallow: /analytics/
# Content Signals (https://contentsignals.org/) — inside the * group so it
# applies to all agents: we explicitly allow search, AI input (grounding/RAG),
# and AI training.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ── AI crawlers explicitly welcome (current UA names, July 2026) ──
# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /analytics/

User-agent: OAI-SearchBot
Allow: /
Disallow: /analytics/

User-agent: ChatGPT-User
Allow: /
Disallow: /analytics/

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /analytics/

User-agent: Claude-User
Allow: /
Disallow: /analytics/

User-agent: Claude-SearchBot
Allow: /
Disallow: /analytics/

User-agent: anthropic-ai
Allow: /
Disallow: /analytics/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /analytics/

User-agent: Perplexity-User
Allow: /
Disallow: /analytics/

# Google (AI grounding / Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /analytics/

# Others
User-agent: Applebot-Extended
Allow: /
Disallow: /analytics/

User-agent: DuckAssistBot
Allow: /
Disallow: /analytics/

User-agent: Amazonbot
Allow: /
Disallow: /analytics/

User-agent: cohere-ai
Allow: /
Disallow: /analytics/

User-agent: Bytespider
Allow: /
Disallow: /analytics/

# Sitemap location
Sitemap: https://andrew.ooo/sitemap.xml

# Machine-readable content indexes for LLMs and agents
# Curated index:   https://andrew.ooo/llms.txt
# Full-text feed:  https://andrew.ooo/llms-full.txt
# Raw Markdown:    /posts/<slug>.md and /answers/<slug>.md (page URL with the trailing slash replaced by .md)