# robots.txt for https://www.taus.net/ # # Files in apps/corporate-ui/public/ are served at the site root and take # precedence over the Nuxt [...slug] catch-all, so this file is what # crawlers see at /robots.txt. # --------------------------------------------------------------------------- # Default rules — apply to every well-behaved crawler # --------------------------------------------------------------------------- User-agent: * # Authenticated user areas (dashboard, sign-in/up, account, billing, data) Disallow: /user/ # Checkout flow (e-commerce, payment intents, error landing) Disallow: /checkout/ # Organization invitations — URLs carry single-use tokens Disallow: /organization/ # Product SKU launcher — redirect-only, no indexable content Disallow: /products/ # Server API endpoints — never useful in search results Disallow: /api/ # Nuxt build artifacts (hashed JS/CSS) — long-lived and not content Disallow: /_nuxt/ # Storyblok visual-editor / draft preview parameters Disallow: /*?_storyblok Disallow: /*&_storyblok Disallow: /*?_storyblok_published Disallow: /*&_storyblok_published # Tracking-parameter URLs — keep canonicals tidy Disallow: /*?utm_ Disallow: /*&utm_ Disallow: /*?gclid= Disallow: /*&gclid= Disallow: /*?fbclid= Disallow: /*&fbclid= Disallow: /*?li_fat_id= Disallow: /*&li_fat_id= # --------------------------------------------------------------------------- # AI / LLM training crawlers # # TAUS sells data-for-AI services, so blanket-blocking these is a business # decision. The block is OFF by default so AI search bots can surface TAUS # content. To opt out of model-training scrapes, uncomment the block below. # Note: robots.txt groups do NOT inherit — each User-agent needs its own # Disallow lines. # --------------------------------------------------------------------------- # User-agent: GPTBot # Disallow: / # # User-agent: ChatGPT-User # Disallow: / # # User-agent: OAI-SearchBot # Disallow: / # # User-agent: Google-Extended # Disallow: / # # User-agent: anthropic-ai # Disallow: / # # User-agent: ClaudeBot # Disallow: / # # User-agent: Claude-Web # Disallow: / # # User-agent: CCBot # Disallow: / # # User-agent: PerplexityBot # Disallow: / # # User-agent: Bytespider # Disallow: / # # User-agent: Amazonbot # Disallow: / # # User-agent: meta-externalagent # Disallow: / # # User-agent: FacebookBot # Disallow: / # # User-agent: cohere-ai # Disallow: / # # User-agent: Applebot-Extended # Disallow: / # --------------------------------------------------------------------------- # Sitemap # --------------------------------------------------------------------------- Sitemap: https://www.taus.net/sitemap.xml