#Robots.txt
User-agent: Wget
Disallow: /

User-agent: *
Disallow: /cgi-bin/    # keep robots out of the executable tree