User-agent: * Sitemap: https://yechte.com/sitemap.xml ############## # custom # ############## # --------------------------------------------------- # Google Priority Crawlers # --------------------------------------------------- User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Mobile Allow: / # --------------------------------------------------- # General rules for ALL crawlers # --------------------------------------------------- User-agent: * # Block admin/backend/private areas Disallow: /web/ Disallow: /website/backend/ Disallow: /mail/ Disallow: /calendar/ Disallow: /im_livechat/ Disallow: /portal/ Disallow: /payment/ Disallow: /auth_signup/ Disallow: /web/login Disallow: /web/signup Disallow: /web/reset_password # Disallow shop behaviours Disallow: /shop/cart Disallow: /shop/checkout Disallow: /shop/search Disallow: /shop/page/ # Block user pages + pagination Disallow: /my/ Disallow: */page/* # Allow static assets (important for CDN) Allow: /web/content/ Allow: /web/static/ Allow: /yechte_website/static/ Allow: https://productionyechtecdn-b20e.kxcdn.com/ # Block query parameter variations to avoid duplicate crawls Disallow: /*? # Sitemap Sitemap: https://yechte.com/sitemap.xml