# robots.txt template version: 2026-08-16-1 # domain: forthclear.io # Source of truth: services/robots_txt_template.py — do not hand-edit. # Task #819 — keep AI brand-facts well-known reachable User-agent: Googlebot Disallow: /.well-known/ # --- AI Search Engines (Tier 1 - drive direct citations) --- # Anthropic three-bot policy split (Task #882 / G09): we list ClaudeBot (training), claude-web (legacy retrieval) and Anthropic-ai (current Claude.ai retrieval) as separate stanzas so Anthropic's bot identity changes do not silently disable our brand indexing. User-agent: CCBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: GPTBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: OAI-SearchBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: ChatGPT-User Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: PerplexityBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Perplexity-User Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: ClaudeBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: claude-web Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Anthropic-ai Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Claude-SearchBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Claude-User Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Google-Extended Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: GoogleOther Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Applebot-Extended Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Google-Agent Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: KagiBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: LinkUpBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: TigrisBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: YouBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md # --- AI Assistants & Agents (Tier 2) --- User-agent: Amazonbot Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Meta-ExternalAgent Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Meta-ExternalFetcher Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: DuckAssistBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: MistralAI-User Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Operator Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: DeepSeekBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: AllenGPT Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Scrapy Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md # --- AI Indexers & Discovery (Tier 3) --- User-agent: Bytespider Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: cohere-ai Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: PhindBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Bravebot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: PetalBot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md User-agent: Diffbot Allow: / Allow: /.well-known/brand-facts.json Allow: /llms.txt Allow: /llms-full.txt Allow: /facts.json Allow: /faq-feed.json Allow: /faq-feed.xml Allow: /agent.json Allow: /AGENTS.md # --- Block AI training-only crawlers (no search benefit) --- User-agent: img2dataset Disallow: / User-agent: cohere-training-data-crawler Disallow: / # --- Bingbot: explicit allow so it can render CSS/JS for quality assessment --- User-agent: Bingbot Allow: / Disallow: /.well-known/ # --- Default rules (preserve Unicorn Platform defaults) --- User-agent: * Disallow: /.well-known/ Disallow: /.env* Disallow: /*.bak$ Disallow: /*.config$ Disallow: /config.json Disallow: /appsettings.json Disallow: /login Disallow: /login?* Disallow: /logout Disallow: /*?next= Disallow: /*.png$ Disallow: /*.jpg$ Disallow: /*.php$ Disallow: /*.gif$ Disallow: /*.jpeg$ Disallow: /*.bmp$ Disallow: /*.svg$ Disallow: /*.webp$ Disallow: /*.ico$ # --- AI / LLM artifacts (Task #955 / #968) --- # llms.txt: https://forthclear.io/llms.txt # llms-full.txt: https://forthclear.io/llms-full.txt # brand-facts.json: https://forthclear.io/.well-known/brand-facts.json # facts.json (alias): https://forthclear.io/facts.json # faq-feed.json: https://forthclear.io/faq-feed.json # faq-feed.xml (RSS): https://forthclear.io/faq-feed.xml # agent.json: https://forthclear.io/agent.json Sitemap: https://forthclear.io/sitemap.xml