User-agent: DuckDuckBot
Crawl-delay: 5
 
User-agent: Applebot
Crawl-delay: 5
 
User-agent: YandexBot
Crawl-delay: 10
 
# --- SEO / marketing crawlers: heavy delay (they hammer servers) ---
User-agent: AhrefsBot
Crawl-delay: 30
 
User-agent: SemrushBot
Crawl-delay: 30
 
User-agent: MJ12bot
Crawl-delay: 30
 
User-agent: DotBot
Crawl-delay: 30
 
User-agent: BLEXBot
Crawl-delay: 30

# --- Everything else: default rules with conservative delay ---
User-agent: *
Crawl-delay: 10
# Common paths that should not be crawled (adjust per site)
Disallow: /admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /search/
Disallow: /*?s=
Disallow: /*?orderby=
Disallow: /*?filter_
Disallow: /*filtering=1
Disallow: /*add-to-cart=*
Disallow: /*add_to_wishlist=*