User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /page-image-checking
Disallow: /author/*
Disallow: /blogs/tag/
Disallow: */tag/*
Disallow: *?taxonomy=*
Disallow: *?wordfence_*
Disallow: /.htaccess
Disallow: /search/
Disallow: */page/*
Disallow: */amp/*

Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10

#Known crawlers
User-agent: LinkedInBot
User-agent: bingbot
User-agent: RSiteAuditor
User-agent: SiteAuditBot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SemrushBot-SI
User-agent: SemrushBot-BA
User-agent: SemrushBot-SWA
User-agent: SemrushBot-CT
User-agent: SplitSignalBot
User-agent: SemrushBot-COUB
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
Crawl-delay: 60

#Blocked crawlers
User-agent: RyteBot
User-agent: OnPageBot
User-agent: dotbot
User-agent: rogerbot
user-agent: MJ12bot
User-agent: SeznamBot
User-agent: DataForSeoBot
User-agent: PetalBot
User-agent: coccocbot-web
User-agent: SEOkicks
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: SerendeputyBot
User-agent: MojeekBot
User-agent: Scrapy
User-agent: Orbbot
User-agent: MauiBot
User-Agent: Neevabot
User-agent: x09Mozilla
User-agent: x22Mozilla
Disallow: /

Sitemap: http://www.bigscal.com/sitemap_index.xml