# robots.txt for tempoparis.com # Generated dynamically to control crawler behavior # Default rules for all bots User-agent: * # Rate limit: wait 2 seconds between requests Crawl-delay: 2 # Allow main pages Allow: /$ Allow: /shop/$ Allow: /shop/womens_apparel/$ Allow: /shop/basixblacklabel/$ Allow: /shop/details/ # Disallow filter parameter combinations (creates infinite URLs) Disallow: /*?*color= Disallow: /*?*size= Disallow: /*?*style= Disallow: /*?*occassion= Disallow: /*?*price= Disallow: /*?*availability= # Disallow deep pagination (page 6+) Disallow: /*?*page=6 Disallow: /*?*page=7 Disallow: /*?*page=8 Disallow: /*?*page=9 Disallow: /*page=[1-9][0-9] # Disallow malformed view_by URLs Disallow: /shop/view_by/ # Disallow admin areas Disallow: /admin/ Disallow: /django_admin/ # Disallow API endpoints Disallow: /api/ # Disallow utility pages Disallow: /health/ Disallow: /readiness/ # ------------------------------------------- # Pinterest Bot - VERY aggressive, needs strict limits # ------------------------------------------- User-agent: Pinterestbot Crawl-delay: 10 # Only allow product detail pages Allow: /shop/details/ # Disallow all listing pages entirely Disallow: /shop/womens_apparel/ Disallow: /shop/basixblacklabel/ Disallow: /shop/view_by/ Disallow: /*? # ------------------------------------------- # Facebook/Meta Crawler # ------------------------------------------- User-agent: facebookexternalhit Crawl-delay: 5 Disallow: /*?*page= Disallow: /*?*color= Disallow: /*?*size= User-agent: meta-externalagent Crawl-delay: 5 Disallow: /*?*page= Disallow: /*?*color= Disallow: /*?*size= # ------------------------------------------- # Amazon Bot # ------------------------------------------- User-agent: Amazonbot Crawl-delay: 5 Disallow: /*?*page= Disallow: /*?*color= Disallow: /*?*size= # ------------------------------------------- # Google (be more permissive for SEO) # ------------------------------------------- User-agent: Googlebot Crawl-delay: 1 # Allow product pages Allow: /shop/details/ # Limit filter combinations Disallow: /*?*page=[6-9] Disallow: /*?*page=[1-9][0-9] # ------------------------------------------- # Bing # ------------------------------------------- User-agent: Bingbot Crawl-delay: 2 # ------------------------------------------- # SEO Tools (Semrush, Ahrefs) # ------------------------------------------- User-agent: SemrushBot Crawl-delay: 10 Disallow: /shop/ User-agent: AhrefsBot Crawl-delay: 10 Disallow: /shop/ # Sitemap location Sitemap: https://www.tempoparis.com/sitemap.xml