# Allow all ad bots to crawl the entire site, including SERPs

# Google Ads Bot
User-agent: AdsBot-Google
Disallow:

# Bing Ads Bot
User-agent: AdIdxBot
Disallow:

# Yahoo! Ads Bot
User-agent: Yahoo Ad Monitoring
Disallow:

# Exclude our SERPs for search bots

# Google Search Bot (this is a duplicate of the * section because google ignores it)
User-agent: googlebot
Disallow: /search/
Disallow: /web/
Disallow: /images/
Disallow: /video/
Disallow: /news/
Disallow: /shopping/
Disallow: /*/search/
Disallow: /*/web/
Disallow: /*/images/
Disallow: /*/video/
Disallow: /*/news/
Disallow: /*/shopping/

# All other bots
User-agent: *
Disallow: /search/
Disallow: /web/
Disallow: /images/
Disallow: /video/
Disallow: /news/
Disallow: /shopping/
Disallow: /*/search/
Disallow: /*/web/
Disallow: /*/images/
Disallow: /*/video/
Disallow: /*/news/
Disallow: /*/shopping/