# search crawler delay set

User-agent: search.marginalia.nu
Crawl-delay: 0

User-agent: MojeekBot
Crawl-delay: 0

User-agent: Yandex
Crawl-delay: 10

# blocking? non-search crawlers that regularly hit the site

User-agent: SemrushBot
Disallow: /

User-agent: Ahrefs
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: MJ12bot
Crawl-delay: 30

User-agent: MegaIndex.ru
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

# general rules to follow for all bots

User-agent: *
Crawl-delay: 5
Disallow: /ajax.php
Disallow: /attachment.php
Disallow: /calendar.php
Disallow: /cron.php
Disallow: /editpost.php
Disallow: /global.php
Disallow: /image.php
Disallow: /inlinemod.php
Disallow: /joinrequests.php
Disallow: /login.php
Disallow: /member.php
Disallow: /memberlist.php
Disallow: /misc.php
Disallow: /moderator.php
Disallow: /newattachment.php
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /online.php
Disallow: /poll.php
Disallow: /postings.php
Disallow: /printthread.php
Disallow: /private.php
Disallow: /profile.php
Disallow: /register.php
Disallow: /report.php
Disallow: /reputation.php
Disallow: /search.php
Disallow: /sendmessage.php
Disallow: /showgroups.php
Disallow: /subscription.php
Disallow: /threadrate.php
Disallow: /usercp.php
Disallow: /usernote.php