# Production Robots.txt file

# Sitemap
Sitemap: https://www.shopstyle.co.uk/sitemap-index.https.xml

User-agent: YandexBot
Crawl-delay: 10

# Baidu doesnt support Crawl-delay but added anyways in case they ever do
User-agent: Baiduspider
Crawl-delay: 10

User-agent: EtaoSpider
Disallow: /

User-agent: Facebot
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: PetalBot
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Pinterestbot
Allow: /action/apiVisitRetailer
Allow: /interstitial/apiVisitRetailer
Crawl-delay: 0.2

User-agent: *
Disallow: /util/request-headers
Disallow: /*wenhao*
Disallow: /trend/
Disallow: /action/
Disallow: /search
Disallow: /*crawl=no*
Disallow: /users
Disallow: /lists
Disallow: /layouts
Disallow: /track
Disallow: /collective/
Disallow: /featured-looks
Disallow: /l/
Disallow: /interstitial/
Disallow: /api/
Disallow: /*max-price*
Disallow: /*?*preventRedirect=*