# Block access to specific groups of pages User-agent: * Disallow: /cart Disallow: /checkout Disallow: /my-account # Allow search crawlers to discover the sitemap Sitemap: https://www.jacadi.us/sitemap.xml # Block CazoodleBot as it does not present correct accept content headers User-agent: CazoodleBot Disallow: / # Block MJ12bot as it is just noise User-agent: MJ12bot Disallow: / # Block dotbot as it cannot parse base urls properly User-agent: dotbot/1.0 Disallow: / # Block Gigabot User-agent: Gigabot Disallow: / #ticket 2644674 # Block Googlebot #User-agent: Googlebot/2.1 #Disallow: / #ticket 2644674 END # Block Filters User-agent: * Disallow: *q=* Allow: *iframe=false&q=* Disallow: *text* Disallow: *productSize* # Allow Filters Allow :*page=*