# robots.txt — see https://developers.google.com/search/docs/crawling-indexing/robots/intro

# Rules for all crawlers (also applies to Google, Bing, etc.)
User-agent: *
Disallow: /user_content/
Disallow: /an/
Disallow: /out/
Disallow: /ajax/

# Blocked crawlers/scrapers (SEO competitors, archivers, generic download libraries)
User-agent: AhrefsBot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Wget
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

# Sitemaps (one robots.txt is served for all locale domains, so absolute URLs are required)
Sitemap: https://www.hosttest.de/sitemap.xml
Sitemap: https://www.hosttest.at/sitemap-at.xml
Sitemap: https://www.hosttest.ch/sitemap-ch.xml
Sitemap: https://www.hosttest.co.uk/sitemap-uk.xml