User-agent: Googlebot Allow: /images/ Allow: /css/ Allow: /js/ User-agent: Bingbot Allow: /images/ Allow: /css/ Allow: /js/ User-agent: "Yahoo Slurp" Allow: /images/ Allow: /css/ Allow: /js/ User-agent: Yandexbot Allow: /images/ Allow: /css/ Allow: /js/ User-agent: DuckDuckBot Allow: /images/ Allow: /css/ Allow: /js/ User-agent: Applebot Allow: /images/ Allow: /css/ Allow: /js/ User-agent: "Facebook External Hit" Allow: /images/ Allow: /css/ Allow: /js/ User-agent: Twitterbot Allow: /images/ Allow: /css/ Allow: /js/ User-agent: * Disallow: /private/ # Disallow access to private directories Disallow: /admin/ # Disallow access to administrative pages Disallow: /cgi-bin/ # Disallow access to CGI scripts Disallow: /tmp/ # Disallow access to temporary directories Disallow: /images/ # Disallow access to images directory Disallow: /css/ # Disallow access to CSS files Disallow: /js/ # Disallow access to JavaScript files Disallow: /downloads/ # Disallow access to downloadable files Disallow: /restricted/ # Disallow access to restricted content