# Sitemaps
Sitemap: https://www.thewrap.com/sitemap.xml
Sitemap: https://www.thewrap.com/news-sitemap.xml
Sitemap: https://www.thewrap.com/duration-PT24H-sitemap.xml
Sitemap: https://www.thewrap.com/duration-PT48H-sitemap.xml
Sitemap: https://www.thewrap.com/duration-PT72H-sitemap.xml
Sitemap: https://www.thewrap.com/duration-P7D-sitemap.xml
Sitemap: https://www.thewrap.com/duration-P30D-sitemap.xml

# General crawl settings
User-agent: *
# Block search queries
Disallow: /?s=*
Disallow: /search/*
# Block wp-admin and the login page
Disallow: /wp-admin/
Disallow: /wp-login.php
# Block Zephr paths
Disallow: /blaize/
Disallow: /zephr/
# Block feeds
Disallow: /feed/
# Block tag pages
Disallow: /tag/
# Block common spammy paths that would otherwise 404
Disallow: /cgi-bin/
Disallow: /*?preview=
Disallow: /*&preview=
Disallow: /*?theme_preview=
Disallow: /*&theme_preview=
# Block magic links from newsletters
Disallow: /*?wtid=

# Block specific bots
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Baiduspider
User-agent: Barkrowler
User-agent: BLEXBot
User-agent: CazoodleBot
User-agent: DataForSeoBot
User-agent: DotBot
User-agent: dotbot/1.0
User-agent: Gigabot
User-agent: Go-http-client
User-agent: Linespider
User-agent: magpie-crawler
User-agent: MojeekBot
User-agent: Node/simplecrawler
User-agent: Nuclei
User-agent: Perplexity-ai
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: PiplBot
User-agent: ProRataInc
User-agent: Riddler
User-agent: Scrapy
User-agent: SeznamBot
User-agent: Sidetrade indexer bot
User-agent: Sogou Spider
User-agent: Timpibot
User-agent: TurnitinBot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: WikiDo
User-agent: Yandexbot
User-agent: YouBot
User-agent: Zoominfobot
User-agent: archive.is
User-agent: archive.today
User-agent: archive.ph
Disallow: /


# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

# ---------------------------
# END YOAST BLOCK