User-agent: *
Crawl-Delay: 60
Disallow: /cgi-bin/
User-agent: psbot
Disallow: /
Allow: /pages
User-agent: Teoma
Disallow: /
Allow: /pages