User-Agent: htdig-rfc
Allow: /ftp-home/
User-Agent: googlebot

User-Agent: WebCrawler

User-Agent: *
Disallow: /ftp-home/
Disallow: /cgi-bin/
Disallow: /tmp/