# Some bots are known to be trouble, particularly those designed to copy entire sites.
#
# Based on the Wikipedia.org list of troublesome bots: https://en.wikipedia.org/robots.txt
#
User-agent: ia_archiver
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: fast
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: k2spider
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: WebReaper
Disallow: /

#
# Other user agents are fine and should work as intended
#
User-agent: *
Crawl-delay: 3
Disallow: /mysqlAdmin/
Disallow: /SpryAssets/
Disallow: /var/www/html/
Disallow: /?blackhole
Disallow: /sponsor_taxonomy/
Disallow: /investment_theme_taxonomy/

Allow: /wp-content/uploads
Allow: /wp-includes/js/jquery
Allow: /wp-includes/js/jquery/jquery-migrate.min.js?ver=1.4.1

Sitemap: https://www.etftrends.com/sitemap_index.xml