# the following settings apply to all bots
User-agent: * 
Disallow: /bin/

User-agent: *
Crawl-delay: 10

# the following settings apply to Baiduspider which is affecting our bandwidth
User-agent: Baiduspider-image
Disallow: /Images/
Disallow: /images/
Disallow: /Downloads/

# the following settings apply to Baiduspider which is affecting our bandwidth
User-agent: Baiduspider
Disallow: /Images/
Disallow: /images/
Disallow: /Downloads/

# Limit ScoutJet's crawl rate (example is to crawl no more than 1 page every 5 seconds)
User-agent: ScoutJet
Crawl-delay: 60

User-agent: MJ12bot
Disallow: /

User-agent: Applebot
Crawl-delay: 10

User-agent: PetalBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: Amazonbot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: Googlebot
Crawl-delay: 10

User-agent: Bingbot
Crawl-delay: 10

User-agent: Thinkbot
Crawl-delay: 10

User-agent: meta-externalagent
Disallow: 

User-agent: DotBot
Crawl-delay: 10

User-agent: GPTBot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

# Block aggressive/unnecessary bots
User-agent: Bytespider
Disallow: /