# robots.txt — last reviewed 2026-10-10
# Owner: a@artka.dev. Policy: allow retrieval/answer crawlers; disallow private surfaces.
# /admin/ and /api/ are private surfaces. Utility HTML remains crawlable so its
# noindex directive can be observed.
#
# Agent resources: https://artka.dev/llms.txt (index) and
# https://artka.dev/llms-full.txt (full-text digest, noindex but fetchable).
# Every post also has a Markdown twin at <page-url minus slash>.md.
# The twins are crawlable for every agent on purpose: each one answers with
# `X-Robots-Tag: noindex` and `Link: <html url>; rel="canonical"`, and a
# crawler can only see those headers if robots.txt lets it fetch the twin.

User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: meta-externalagent
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: DuckAssistBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: MistralAI-User
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Diffbot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/

Sitemap: https://artka.dev/sitemap-index.xml