User-agent: Mediapartners-Google
Allow: /
Disallow: /cgi-bin/links/go.cgi
Disallow: /cgi-bin/links/go3.cgi
Disallow: /cgi-bin/links/modify.cgi
Disallow: /cgi-bin/links/gethtml.cgi
Disallow: /cgi-bin/links/rateit.cgi
Disallow: /cgi-bin/links/rateit2.cgi
Disallow: /cgi-bin/links/rprtb.cgi
Disallow: /cgi-bin/links/rprtb2.cgi
Disallow: /cgi-bin/links/addurl.cgi
Disallow: /cgi-bin/meta/
Disallow: /cgi-bin/files/
Disallow: /cgi-bin/welcome/
Disallow: /cgi-bin/services/
Disallow: /cgi-bin/asp/
Disallow: /cgi-bin/directories/
Disallow: /cgi-bin/resources/
Disallow: /cgi-bin/pass/
Disallow: /cgi-bin/mass/
Disallow: /spool/
Disallow: /wap/
Disallow: /files/
Disallow: /directories/
Disallow: /welcome/


User-agent: Googlebot
Disallow: /cgi-bin/
Disallow: /spool/
Disallow: /wap/
Disallow: /files/
Disallow: /directories/
Disallow: /welcome/


User-agent: Slurp
Crawl-delay: 90
Disallow: /cgi-bin/

User-agent: bingbot
Crawl-delay: 30
Disallow: /cgi-bin/

User-agent: *
Disallow: /