# ---------------------------------------------------
# CRAWLERS/BOTS NOT ALLOWED
# ---------------------------------------------------

# Digg Mirror (Blocks Digg crawler)
User-agent: duggmirror
Disallow: /

# Alexa archiver
User-agent: ia_archiver
Disallow: /

# HTTrack
User-agent: HTTrack
Disallow: /

# libwww Perl
User-agent: libwww 
Disallow: /

# MSIECrawler bot
User-agent: MSIECrawler 
Disallow: / 

# WebCopier 
User-agent: WebCopier 
Disallow: /

# WebZIP 
User-agent: WebZIP 
Disallow: /

# Spbot
User-agent: spbot
Disallow: /

# ---------------------------------------------------
# CRAWLERS/BOTS ALLOWED
# ---------------------------------------------------

# Twitterbot
User-agent: Twitterbot
Disallow: 

# Google Image
User-agent: Googlebot-Image
Disallow:

# Google AdSense
User-agent: Mediapartners-Google
Disallow:

# Googlebot
User-agent: Googlebot
Disallow: /cgi-bin/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Rules for the rest of crawlers/bots
User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /wp-content/languages/
Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php

# ---------------------------------------------------
# SITEMAP
# ---------------------------------------------------

Sitemap: https://www.elarraydejota.com/sitemap.xml