# robots.txt for completeaibusiness.com # # The policy for crawlers and AI agents lives here, in the repo (site/public/robots.txt), not in Cloudflare's # Managed robots.txt, so a dashboard toggle can't change or empty it. # # Content-Signal (https://contentsignals.org, draft-romm-aipref-contentsignals) says how the public pages may be used: # search = yes: appear in search results # ai-input = yes: be read to answer a person's question (retrieval, grounding) # ai-train = no: not used to train or fine-tune models # To change the policy, change the Content-Signal line (it is the same in every block), e.g. ai-train=yes to allow training. # # Every crawler, named or not, gets the same rules. Plan pages (/report), the API and the sign-in pages are private # and stay out. Nothing here is enforced at the edge. # # ANY RESTRICTION EXPRESSED VIA CONTENT SIGNALS IS AN EXPRESS RESERVATION OF RIGHTS UNDER ARTICLE 4 OF # EU DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET. User-agent: * Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ # AI search and assistants fetching a page for a person User-agent: OAI-SearchBot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: ChatGPT-User Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Claude-SearchBot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Claude-User Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: PerplexityBot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Perplexity-User Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: DuckAssistBot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: MistralAI-User Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Applebot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ # AI training crawlers: welcome to read for search and answers, asked not to train (ai-train=no) User-agent: GPTBot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: ClaudeBot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: anthropic-ai Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Google-Extended Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Applebot-Extended Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: CCBot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Bytespider Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Amazonbot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: meta-externalagent Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: meta-externalfetcher Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: cohere-ai Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: PetalBot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Diffbot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ User-agent: Timpibot Content-Signal: search=yes, ai-input=yes, ai-train=no Allow: / Disallow: /report Disallow: /api/ Disallow: /oauth/ Sitemap: https://completeaibusiness.com/sitemap.xml # Where agents find this site's MCP server, A2A agent, API and skill (ARD) Agentmap: https://completeaibusiness.com/.well-known/ai-catalog.json