User-agent: *
Crawl-delay: 30 

# robots.txt
User-agent: ia_archiver
Disallow: /

User-agent: scooter
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /