# # robots.txt # # version 1.0 # # exclude User-Agent: * Disallow: /images/ Disallow: /forum/viewtopic.php Disallow: /forum/viewforum.php Disallow: /forum/index.php? Disallow: /forum/posting.php Disallow: /forum/search.php Disallow: /forum/ucp.php Disallow: /forum/mcp.php Disallow: /forum/post Disallow: /forum/member Disallow: /forum/memberlist.php Disallow: /forum/faq.php Disallow: /forum/download/file.php Disallow: /forum/report.php Disallow: /forum/cron.php Disallow: /set-blocking-cookie.php User-agent: Googlebot-Image Disallow: / User-agent: DIIbot Disallow: / User-agent: LinkWalker Disallow: / User-agent: NPBot Disallow: / User-agent: BaySpider Disallow: / # Genealogy SE http://www.GenDoor.com/crawler.html User-agent: GenCrawler Disallow: / # http://www.gozilla.com/ User-agent: Go!Zilla Disallow: / # Bad Links User-agent: Linkidator Disallow: / # Bad Links http://www.elsop.com/linkscan/overview.html User-agent: LinkScan Disallow: / # Link Checking http://www.linkguard.com/ User-agent: LinkGuard Disallow: / User-agent: TurnitinBot Disallow: / Sitemap: http://www.slurls.com/sitemap_index.xml