# MFG Calcs
User-agent: *
Allow: /
# Social card images must stay fetchable or link previews and Discover break.
Allow: /api/og
Allow: /api/og?*
Allow: /api/workbook
Allow: /api/workbook/*
Disallow: /api/
Disallow: /version
# Query strings: only the parameter spaces that cost something are closed.
# /?q= and /search?q= make middleware.ts fetch /api/answer to build a
# crawler share card; /news?desk= is a filtered view of the news hub.
# Every other query variant (shared calculator state, ?compare=, ?asOf=,
# UTM tags) serves the same static HTML with its self-canonical, so a
# crawler may fetch it and fold it into the canonical URL.
Disallow: /?
Disallow: /search?
Disallow: /news?
# Share links carry their channel in the query (the hash holds share
# anchors). These longer rules beat the Disallows above on every engine
# that applies longest-match (Google, Bing).
Allow: /*?mfg_share=
Allow: /*&mfg_share=
# Honeypot: linked invisibly in the footer and disallowed here. A
# compliant crawler under this * block never fetches it; anything that
# does is ignoring these rules (docs/PLAN_BOT_DOSSIER.md). The named AI
# engines below have their own allow-all blocks and are out of scope,
# which is fine: they are the verified population the trap is not for.
Disallow: /crawler-trap
# Worked-example scenario pages: noindex, not prerendered, and therefore
# a 404 for anything that does not run JavaScript. Nothing gains by
# crawling them, so every agent is told to skip them.
Disallow: /calculators/*/examples/

# AI and answer engines are welcome to read and cite our content.
User-agent: GPTBot
Allow: /
Disallow: /calculators/*/examples/

User-agent: OAI-SearchBot
Allow: /
Disallow: /calculators/*/examples/

User-agent: ChatGPT-User
Allow: /
Disallow: /calculators/*/examples/

User-agent: ClaudeBot
Allow: /
Disallow: /calculators/*/examples/

User-agent: Claude-Web
Allow: /
Disallow: /calculators/*/examples/

User-agent: anthropic-ai
Allow: /
Disallow: /calculators/*/examples/

User-agent: Claude-SearchBot
Allow: /
Disallow: /calculators/*/examples/

User-agent: Claude-User
Allow: /
Disallow: /calculators/*/examples/

User-agent: PerplexityBot
Allow: /
Disallow: /calculators/*/examples/

User-agent: Perplexity-User
Allow: /
Disallow: /calculators/*/examples/

User-agent: Google-Extended
Allow: /
Disallow: /calculators/*/examples/

User-agent: Applebot-Extended
Allow: /
Disallow: /calculators/*/examples/

User-agent: CCBot
Allow: /
Disallow: /calculators/*/examples/

User-agent: Amazonbot
Allow: /
Disallow: /calculators/*/examples/

User-agent: cohere-ai
Allow: /
Disallow: /calculators/*/examples/

User-agent: Bytespider
Allow: /
Disallow: /calculators/*/examples/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /calculators/*/examples/

User-agent: DuckAssistBot
Allow: /
Disallow: /calculators/*/examples/

User-agent: MistralAI-User
Allow: /
Disallow: /calculators/*/examples/

# LLM discovery: https://www.mfgcalcs.com/llms.txt and https://www.mfgcalcs.com/llms-full.txt
Sitemap: https://www.mfgcalcs.com/sitemap.xml
Sitemap: https://www.mfgcalcs.com/sitemap-index.xml