# https://moz.com/help/guides/moz-procedures/what-is-rogerbot
User-agent: rogerbot
Disallow: /

# http://help.coccoc.com/
User-agent: coccoc
Disallow: /

# ubermetrics-technologies.com
User-agent: um-IC
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: coccocbot-web
Disallow: /

User-Agent: YisouSpider
Disallow: /

User-Agent: tra.cx.pider
Disallow: /

User-Agent: ConveraCrawler 
Disallow: /

User-Agent: kalooga
Disallow: /

User-Agent: BecomeBot
Disallow: /

User-Agent: EasyDL
Disallow: /

User-agent: Cursor
Disallow: /

User-Agent: Nutch
Disallow: /

User-Agent: Fasterfox
Disallow: /

User-Agent: psbot
Disallow: /

User-Agent: pagebullbot
Disallow: /

User-Agent: RufusBot
Disallow: /

User-Agent: SumeetBot
Disallow: /

User-Agent: twiceler
Disallow: /

User-Agent: woriobot
Disallow: /

# Ignored
User-agent: MJ12bot
Disallow: /

# Ignored
User-agent: Yandex
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Blekkobot
Disallow: /

User-agent: ScoutJet
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: dataforseo-bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Bytespider
Disallow: / 

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: GPTBot
Disallow: /

User-Agent: *
Crawl-delay: 2

Disallow: /StoryTracking-
Disallow: /StoryInfo-
Disallow: /RelatedStories-
Disallow: /StoryTracking-
Disallow: /AuthorRecommendations-
Disallow: /AuthorReviews-
Disallow: /StoryTracking-
Disallow: /AuthorReviewsWritten-
Disallow: /Track-

Disallow: /addchallenge.php
Disallow: /addreview.php
Disallow: /addseries.php
Disallow: /addstory.php
Disallow: /addto.php
Disallow: /antispam.php
Disallow: /authorreviews.php
Disallow: /bandwidthabuse.php
Disallow: /barimg.php
Disallow: /browse.php
Disallow: /cacheinfo.php
Disallow: /cachetrap.php
Disallow: /coapic.php
Disallow: /contact.php
Disallow: /customise.php
Disallow: /deletechallenge.php
Disallow: /deletechapter.php
Disallow: /deleteffa.php
Disallow: /deletereview.php
Disallow: /deleteseries.php
Disallow: /deletestory.php
Disallow: /editchallenge.php
Disallow: /editchapter.php
Disallow: /editffa.php
Disallow: /editprofile.php
Disallow: /editreview.php
Disallow: /editseries.php
Disallow: /editstorydetails.php
Disallow: /edittracking.php
Disallow: /emailauthor.php
Disallow: /ffaselectchar2.php
Disallow: /error.php
Disallow: /googlesearch.php
Disallow: /hits.kmz
Disallow: /login.php
Disallow: /logout.php
Disallow: /lookup.php
Disallow: /lookupchallenge.php
Disallow: /maxrating.php
Disallow: /myprofile.php
Disallow: /newfeatures.php
Disallow: /pieimg.php
Disallow: /quarantineadd.php
Disallow: /quarantineremove.php
Disallow: /randomfic.php
Disallow: /randomunclassfied.php
Disallow: /recommend.php
Disallow: /recommendedby.php
Disallow: /register.php
Disallow: /report.php
Disallow: /reviewcomment.php
Disallow: /search.php
Disallow: /searchtips.php
Disallow: /setmaxrating.php
Disallow: /shop.php
Disallow: /shop2.php
Disallow: /storyimg.php
Disallow: /storystyles.php
Disallow: /style.php
Disallow: /test.php
Disallow: /userpanel.php
Disallow: /userrecommendedby.php
Disallow: /wholestory.php