User-agent: * Sitemap: https://www.softgoat.com/sitemap.xml Disallow: /selection Disallow: /selection/* # Google Adsbot ignores robots.txt unless specifically named User-agent: adsbot-google Disallow: /selection Disallow: /selection/* # Slow down some bots User-agent: Pinterest Crawl-delay: 1 User-agent: Yandex Crawl-Delay: 10 User-agent: AhrefsBot Crawl-delay: 10 User-agent: AhrefsSiteAudit Crawl-delay: 10 User-agent: MJ12bot Crawl-Delay: 10 # Disallow bad bots User-agent: Nutch Disallow: / User-agent: sitebot Disallow: / User-agent: LexxeBot Disallow: / User-agent: Ezooms Disallow: / User-agent: ichiro Disallow: / User-agent: Alligator Disallow: / User-agent: SQ\ Webscanner Disallow: / User-agent: AllSubmitter Disallow: / User-agent: NaverBot Disallow: / User-agent: Anonymous Disallow: / User-agent: Asterias Disallow: / User-agent: autoemailspider Disallow: / User-agent: Badass Disallow: / User-agent: Baiduspider Disallow: / User-agent: BecomeBot Disallow: / User-agent: Bitacle Disallow: / User-agent: dragonfly Disallow: / User-agent: DreamPassport Disallow: / User-agent: DSurf Disallow: / User-agent: EBrowse Disallow: / User-agent: eCatch Disallow: / User-agent: edgeio Disallow: / User-agent: DreamPassport Disallow: / User-agent: DTS Agent Disallow: / User-agent: FSurf Disallow: / User-agent: Gaisbot Disallow: / User-agent: Gamespy_Arcade Disallow: / User-agent: genieBot Disallow: / User-agent: GetBot Disallow: / User-agent: GetRight Disallow: / User-agent: Gigabot Disallow: / User-agent: Go!Zilla Disallow: / User-agent: Go-Ahead-Got-It Disallow: / User-agent: GOFORITBOT Disallow: / User-agent: Email\ Extractor Disallow: / User-agent: EmailSiphon Disallow: / User-agent: jeteye Disallow: / User-agent: jeteyebot Disallow: / User-agent: JoBo Disallow: / User-agent: JOC\ Web\ Spider Disallow: / User-agent: Kapere Disallow: / User-agent: Larbin Disallow: / User-agent: LeechGet Disallow: / User-agent: LightningDownload Disallow: / User-agent: Linkie Disallow: / User-agent: Mac\ Finder Disallow: / User-agent: Mail\ Sweeper Disallow: / User-agent: OutfoxBot Disallow: / User-agent: OmniExplorer Disallow: / User-agent: P3P Disallow: / User-agent: psycheclone Disallow: / User-agent: puf Disallow: / User-agent: PussyCat Disallow: / User-agent: PuxaRapido Disallow: / User-agent: Python-urllib Disallow: / User-agent: RealDownload Disallow: / User-agent: RedKernel Disallow: / User-agent: relevantnoise Disallow: / User-agent: Y!TunnelPro Disallow: / User-agent: YahooYSMcm Disallow: / User-agent: Zade Disallow: / User-agent: ZBot Disallow: / User-agent: zerxbot Disallow: / # Fix for googlebot User-agent: Googlebot Disallow: User-agent: Googlebot-image Disallow: