# ---------------------------------------------------------------------------
# robots.txt - veda.ch - AyurVeda AG, Rain LU (CH)
# Version 2026-09-14 - prepared by TC Digital Marketing Sarl
# Languages: German = / (root), French = /fr/. Both fully open to indexing.
# ---------------------------------------------------------------------------
# Design notes for the implementer:
#  - A crawler obeys only ONE group: the most specific one matching its name.
#    Every named group below therefore repeats the same hygiene rules.
#    Do not "factor them out", it would silently unblock the noisy URLs.
#  - Crawl-delay is ignored by Google and honoured by Bing and most third
#    party crawlers. It is used here only for non-search crawlers, to protect
#    the origin server.
# ---------------------------------------------------------------------------


# ===========================================================================
# 1. DEFAULT GROUP - applies to every crawler not named below
# ===========================================================================
User-agent: *
Allow: /

# Rendering assets must stay crawlable, otherwise Google sees a broken page
Allow: /static/
Allow: /media/
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$

# Transactional and account areas: no search value, expensive to render
Disallow: /checkout/
Disallow: /customer/
Disallow: /wishlist/
Disallow: /sales/
Disallow: /newsletter/
Disallow: /review/
Disallow: /catalogsearch/
Disallow: /*/checkout/
Disallow: /*/customer/
Disallow: /*/wishlist/
Disallow: /*/sales/
Disallow: /*/newsletter/
Disallow: /*/review/
Disallow: /*/catalogsearch/

# Magento technical paths
Disallow: /downloader/
Disallow: /app/
Disallow: /lib/
Disallow: /var/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /errors/
Disallow: /cron.php
Disallow: /install.php
Disallow: /*.php$

# Faceted navigation and session parameters.
# These multiply a few hundred real pages into tens of thousands of
# near-duplicate URLs. They are the main crawl-budget leak on this site.
Disallow: /*?*SID=
Disallow: /*?*___store=
Disallow: /*?*___from_store=
Disallow: /*?*price=
Disallow: /*?*cat=
Disallow: /*?*manufacturer=
Disallow: /*?*color=
Disallow: /*?*size=
Disallow: /*?*limit=
Disallow: /*?*dir=
Disallow: /*?*order=
Disallow: /*?*mode=
Disallow: /*?*product_list_
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*msclkid=

# Do NOT disallow /*? as a whole: it would also block legitimate pagination
# and any parameterised landing page used by advertising.


# ===========================================================================
# 2. SEARCH ENGINES - explicitly welcomed, both languages
# ===========================================================================
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Storebot-Google
User-agent: GoogleOther
User-agent: Bingbot
User-agent: msnbot
User-agent: msnbot-media
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Qwantbot
User-agent: Seznambot
Allow: /
Allow: /static/
Allow: /media/
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$
Disallow: /checkout/
Disallow: /customer/
Disallow: /wishlist/
Disallow: /sales/
Disallow: /newsletter/
Disallow: /review/
Disallow: /catalogsearch/
Disallow: /*/checkout/
Disallow: /*/customer/
Disallow: /*/wishlist/
Disallow: /*/sales/
Disallow: /*/newsletter/
Disallow: /*/review/
Disallow: /*/catalogsearch/
Disallow: /downloader/
Disallow: /app/
Disallow: /lib/
Disallow: /var/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /errors/
Disallow: /*.php$
Disallow: /*?*SID=
Disallow: /*?*___store=
Disallow: /*?*___from_store=
Disallow: /*?*price=
Disallow: /*?*cat=
Disallow: /*?*manufacturer=
Disallow: /*?*color=
Disallow: /*?*size=
Disallow: /*?*limit=
Disallow: /*?*dir=
Disallow: /*?*order=
Disallow: /*?*mode=
Disallow: /*?*product_list_
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*msclkid=


# ===========================================================================
# 3. AI SEARCH AND ANSWER ENGINES (GEO) - explicitly welcomed
# These are the crawlers that feed ChatGPT Search, Gemini and AI Overviews,
# Perplexity, Claude, Copilot and Apple Intelligence. Their absence from a
# robots.txt already means "allowed", but naming them is deliberate: it
# documents the choice and survives future edits by someone else.
# ===========================================================================
User-agent: Google-Extended
User-agent: OAI-SearchBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: MistralAI-User
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: CCBot
User-agent: cohere-ai
User-agent: YouBot
User-agent: Bytespider
Allow: /
Allow: /static/
Allow: /media/
Disallow: /checkout/
Disallow: /customer/
Disallow: /wishlist/
Disallow: /sales/
Disallow: /newsletter/
Disallow: /review/
Disallow: /catalogsearch/
Disallow: /*/checkout/
Disallow: /*/customer/
Disallow: /*/wishlist/
Disallow: /*/sales/
Disallow: /*/newsletter/
Disallow: /*/review/
Disallow: /*/catalogsearch/
Disallow: /downloader/
Disallow: /app/
Disallow: /lib/
Disallow: /var/
Disallow: /errors/
Disallow: /*.php$
Disallow: /*?*SID=
Disallow: /*?*price=
Disallow: /*?*cat=
Disallow: /*?*limit=
Disallow: /*?*dir=
Disallow: /*?*order=
Disallow: /*?*mode=
Disallow: /*?*product_list_
Disallow: /*?*utm_
Crawl-delay: 2


# ===========================================================================
# 4. SEO AND MONITORING TOOLS - allowed, rate limited
# AhrefsBot was previously blocked entirely. It is unblocked here because it
# is the independent source we use to measure organic traffic when the site's
# own analytics fails, which has happened twice in three months.
# ===========================================================================
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: DataForSeoBot
User-agent: Screaming Frog SEO Spider
User-agent: SiteAuditBot
Allow: /
Disallow: /checkout/
Disallow: /customer/
Disallow: /wishlist/
Disallow: /sales/
Disallow: /catalogsearch/
Disallow: /*/checkout/
Disallow: /*/customer/
Disallow: /*/wishlist/
Disallow: /*/sales/
Disallow: /*/catalogsearch/
Disallow: /*.php$
Disallow: /*?*SID=
Disallow: /*?*price=
Disallow: /*?*cat=
Disallow: /*?*limit=
Disallow: /*?*dir=
Disallow: /*?*order=
Disallow: /*?*mode=
Disallow: /*?*product_list_
Crawl-delay: 5


# ===========================================================================
# 5. SCRAPERS AND LOW VALUE CRAWLERS - blocked
# These bring no traffic, no citations and no revenue, and they consume
# origin capacity that the site currently does not have to spare.
# ===========================================================================
User-agent: dotbot
User-agent: MJ12bot
User-agent: BLEXBot
User-agent: SEOkicks
User-agent: serpstatbot
User-agent: ZoominfoBot
User-agent: Barkrowler
User-agent: DomainStatsBot
User-agent: Geedo
Disallow: /


# ===========================================================================
# 6. SITEMAPS
# ===========================================================================
Sitemap: https://veda.ch/pub/sitemap_de.xml
Sitemap: https://veda.ch/pub/sitemap_fr.xml
Sitemap: https://veda.ch/pub/sitemap_de.xml
Sitemap: https://veda.ch/pub/sitemap_fr.xml