User-agent: dotbot
Disallow: /

User-Agent: BLEXBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-Agent: SEOkicks-Robot
Disallow: /

User-Agent: archive.org_bot
Disallow: /

User-Agent: special_archiver
Disallow: /

User-Agent: heritrix
Disallow: /

User-Agent: sistrix
Disallow: /

User-Agent: WBSearchBot
Disallow: /

User-agent: proximic
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: *
Disallow: /hub/