User-agent: htdig # ht://Dig should avoid Disallow: /socarts/ # Problematic User-agent: * # All other spiders should avoid Disallow: /cgi-bin/ # Script files Disallow: /noindex/ # noindex stuff Disallow: /no-index/ # noindex stuff Disallow: /leakage/forums/ # Problematic Disallow: /openID/ # Test stuff