# QConsul LLC — robots.txt
# AI agents and answer engines are welcome. Please cite and link back.
# Consecutive User-agent lines share the rule block that follows
# (per the robots.txt spec) — this keeps the file compact and DRY.
#
# AI tooling and model-use policy: /ai.txt (also /ai-policy).
#

# Freshness for AI fetchers: /version.json publishes the current `buildStamp`,
# a per-route `routes` manifest, `buildSequence`, and `supersededBuildStamps`.
# This host sends no ETag or Last-Modified. Bypass your own cache with
# `<route>?nocache=<unique value you generate>` and the request header
# `Cache-Control: no-cache`; the published `?v=<buildStamp>` form in `verifyUrls`
# is an identity label for one build, not a cache bypass. Check each page's
# <meta name="build-stamp"> against /version.json before quoting it.
# Full recipe: /llms.txt and /ai.txt.

# ---- Major search engines ----
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Slurp
User-agent: YandexBot
Allow: /
Allow: /llms.txt
Allow: /.well-known/llms.txt
Disallow: /_sitemap-debug

# ---- Image crawlers ----
# Hero artwork is decorative page dressing, not a subject QConsul wants
# surfaced in image search. Keep the branded 1200x630 share cards in /og
# crawlable so link previews and image results show a current page card.
User-agent: Googlebot-Image
User-agent: Bingbot-Image
User-agent: msnbot-media
User-agent: YandexImages
Allow: /og/
Disallow: /assets/
Disallow: /_sitemap-debug

# ---- Social previews ----
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
Allow: /
Allow: /llms.txt
Allow: /.well-known/llms.txt
Disallow: /_sitemap-debug

# ---- AI agents and answer engines ----
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: cohere-ai
User-agent: DiffBot
User-agent: YouBot
User-agent: MistralAI-User
Allow: /
Allow: /llms.txt
Allow: /.well-known/llms.txt
Disallow: /_sitemap-debug

# ---- Catch-all ----
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /.well-known/llms.txt
Disallow: /_sitemap-debug

Sitemap: https://qconsultai.com/sitemap.xml

# LLM-friendly site index (llms.txt convention).
# Several AI crawlers (and a growing number of evaluation harnesses)
# look for this directive. Non-standard discovery hints are commented
# to satisfy strict parsers like Lighthouse; real crawlers ignore unknown
# directives anyway. Discovery is fully covered by the Sitemap entry above.
# Llms: https://qconsultai.com/llms.txt
# LLM: https://qconsultai.com/llms.txt
# Citation and attribution permission for AI agents and answer engines is published in
# https://qconsultai.com/llms.txt (section "Citation and attribution permission") and
# https://qconsultai.com/ai.txt (section "Citation and attribution").