Robots.txt doesn't show up after WP Defender

Hi there,

The robots.txt file doesn't show up anymore on my website http://www.daizy.media
It should be visible when going to http://www.daizy.media/robots.txt

Does anyone have an solution?

Thanks,

  • Martijn
    • Design Lord, Child of Thor

    # BEGIN WP Rocket v2.7.2
    # Use UTF-8 encoding for anything served text/plain or text/html
    AddDefaultCharset UTF-8
    # Force UTF-8 for a number of file formats
    <IfModule mod_mime.c>
    AddCharset UTF-8 .atom .css .js .json .rss .vtt .xml
    </IfModule>

    <Files robot.txt>
    Allow from all
    </Files>

    # FileETag None is not enough for every server.
    <IfModule mod_headers.c>
    Header unset ETag
    </IfModule>

    # Since we're sending far-future expires, we don't need ETags for static content.
    # developer.yahoo.com/performance/rules.html#etags
    FileETag None

    <IfModule mod_alias.c>
    <FilesMatch "\.(html|htm|rtf|rtx|txt|xsd|xsl|xml)$">
    <IfModule mod_headers.c>
    Header set X-Powered-By "WP Rocket/2.7.2"
    Header unset Pragma
    Header append Cache-Control "public"
    Header unset Last-Modified
    </IfModule>
    </FilesMatch>

    <FilesMatch "\.(css|htc|js|asf|asx|wax|wmv|wmx|avi|bmp|class|divx|doc|docx|eot|exe|gif|gz|gzip|ico|jpg|jpeg|jpe|json|mdb|mid|midi|mov|qt|mp3|m4a|mp4|m4v|mpeg|mpg|mpe|mpp|otf|odb|odc|odf|odg|odp|ods|odt|ogg|pdf|png|pot|pps|ppt|pptx|ra|ram|svg|svgz|swf|tar|tif|tiff|ttf|ttc|wav|wma|wri|xla|xls|xlsx|xlt|xlw|zip)$">
    <IfModule mod_headers.c>
    Header unset Pragma
    Header append Cache-Control "public"
    </IfModule>
    </FilesMatch>
    </IfModule>

    # Expires headers (for better cache control)
    <IfModule mod_expires.c>
    ExpiresActive on

    # Perhaps better to whitelist expires rules? Perhaps.
    ExpiresDefault "access plus 1 month"

    # cache.appcache needs re-requests in FF 3.6 (thanks Remy ~Introducing HTML5)
    ExpiresByType text/cache-manifest "access plus 0 seconds"

    # Your document html
    ExpiresByType text/html "access plus 0 seconds"

    # Data
    ExpiresByType text/xml "access plus 0 seconds"
    ExpiresByType application/xml "access plus 0 seconds"
    ExpiresByType application/json "access plus 0 seconds"

    # Feed
    ExpiresByType application/rss+xml "access plus 1 hour"
    ExpiresByType application/atom+xml "access plus 1 hour"

    # Favicon (cannot be renamed)
    ExpiresByType image/x-icon "access plus 1 week"

    # Media: images, video, audio
    ExpiresByType image/gif "access plus 1 month"
    ExpiresByType image/png "access plus 1 month"
    ExpiresByType image/jpeg "access plus 1 month"
    ExpiresByType video/ogg "access plus 1 month"
    ExpiresByType audio/ogg "access plus 1 month"
    ExpiresByType video/mp4 "access plus 1 month"
    ExpiresByType video/webm "access plus 1 month"

    # HTC files (css3pie)
    ExpiresByType text/x-component "access plus 1 month"

    # Webfonts
    ExpiresByType application/x-font-ttf "access plus 1 month"
    ExpiresByType font/opentype "access plus 1 month"
    ExpiresByType application/x-font-woff "access plus 1 month"
    ExpiresByType application/x-font-woff2 "access plus 1 month"
    ExpiresByType image/svg+xml "access plus 1 month"
    ExpiresByType application/vnd.ms-fontobject "access plus 1 month"

    # CSS and JavaScript
    ExpiresByType text/css "access plus 1 year"
    ExpiresByType application/javascript "access plus 1 year"

    </IfModule>

    # Gzip compression
    <IfModule mod_deflate.c>
    # Active compression
    SetOutputFilter DEFLATE
    # Force deflate for mangled headers
    <IfModule mod_setenvif.c>
    <IfModule mod_headers.c>
    SetEnvIfNoCase ^(Accept-EncodXng|X-cept-Encoding|X{15}|~{15}|-{15})$ ^((gzip|deflate)\s*,?\s*)+|[X~-]{4,13}$ HAVE_Accept-Encoding
    RequestHeader append Accept-Encoding "gzip,deflate" env=HAVE_Accept-Encoding
    # Don't compress images and other uncompressible content
    SetEnvIfNoCase Request_URI \
    \.(?:gif|jpe?g|png|rar|zip|exe|flv|mov|wma|mp3|avi|swf|mp?g)$ no-gzip dont-vary
    </IfModule>
    </IfModule>

    # Compress all output labeled with one of the following MIME-types
    <IfModule mod_filter.c>
    AddOutputFilterByType DEFLATE application/atom+xml \
    application/javascript \
    application/json \
    application/rss+xml \
    application/vnd.ms-fontobject \
    application/x-font-ttf \
    application/xhtml+xml \
    application/xml \
    font/opentype \
    image/svg+xml \
    image/x-icon \
    text/css \
    text/html \
    text/plain \
    text/x-component \
    text/xml
    </IfModule>
    <IfModule mod_headers.c>
    Header append Vary: Accept-Encoding
    </IfModule>
    </IfModule>

    <IfModule mod_mime.c>
    AddType text/html .html_gzip
    AddEncoding gzip .html_gzip
    </IfModule>
    <IfModule mod_setenvif.c>
    SetEnvIfNoCase Request_URI \.html_gzip$ no-gzip
    </IfModule>

    <IfModule mod_rewrite.c>
    RewriteEngine On
    RewriteBase /
    RewriteCond %{HTTPS} on [OR]
    RewriteCond %{SERVER_PORT} ^443$ [OR]
    RewriteCond %{HTTP:X-Forwarded-Proto} https
    RewriteRule .* - [E=WPR_SSL:-https]
    RewriteCond %{HTTP:Accept-Encoding} gzip
    RewriteRule .* - [E=WPR_ENC:_gzip]
    RewriteCond %{REQUEST_METHOD} GET
    RewriteCond %{QUERY_STRING} =""
    RewriteCond %{HTTP:Cookie} !(wordpress_logged_in_|wp-postpass_|wptouch_switch_toggle|comment_author_|comment_author_email_) [NC]
    RewriteCond %{REQUEST_URI} !^(.*/feed/?)$ [NC]
    RewriteCond %{HTTP_USER_AGENT} !^(facebookexternalhit).* [NC]
    RewriteCond "%{DOCUMENT_ROOT}/wp-content/cache/wp-rocket/%{HTTP_HOST}%{REQUEST_URI}/index%{ENV:WPR_SSL}.html%{ENV:WPR_ENC}" -f
    RewriteRule .* "/wp-content/cache/wp-rocket/%{HTTP_HOST}%{REQUEST_URI}/index%{ENV:WPR_SSL}.html%{ENV:WPR_ENC}" [L]
    </IfModule>
    # END WP Rocket

    AddDefaultCharset UTF-8
    <IfModule mod_mime.c>
    AddCharset UTF-8 .atom .css .js .json .rss .vtt .xml
    </IfModule>

    <IfModule mod_headers.c>
    Header unset ETag
    </IfModule>

    FileETag None

    <IfModule mod_alias.c>
    <FilesMatch "\.(html|htm|rtf|rtx|svg|svgz|txt|xsd|xsl|xml)$">
    <IfModule mod_headers.c>
    Header set X-Powered-By "WP Rocket/2.6.16"
    Header unset Pragma
    Header append Cache-Control "public"
    Header unset Last-Modified
    </IfModule>
    </FilesMatch>

    <FilesMatch "\.(css|htc|js|asf|asx|wax|wmv|wmx|avi|bmp|class|divx|doc|docx|eot|exe|gif|gz|gzip|ico|jpg|jpeg|jpe|json|mdb|mid|midi|mov|qt|mp3|m4a|mp4|m4v|mpeg|mpg|mpe|mpp|otf|odb|odc|odf|odg|odp|ods|odt|ogg|pdf|png|pot|pps|ppt|pptx|ra|ram|svg|svgz|swf|tar|tif|tiff|ttf|ttc|wav|wma|wri|xla|xls|xlsx|xlt|xlw|zip)$">
    <IfModule mod_headers.c>
    Header unset Pragma
    Header append Cache-Control "public"
    </IfModule>
    </FilesMatch>
    </IfModule>

    <IfModule mod_expires.c>
    ExpiresActive on

    ExpiresDefault "access plus 1 month"
    ExpiresByType text/cache-manifest "access plus 0 seconds"
    ExpiresByType text/html "access plus 0 seconds"
    ExpiresByType text/xml "access plus 0 seconds"
    ExpiresByType application/xml "access plus 0 seconds"
    ExpiresByType application/json "access plus 0 seconds"
    ExpiresByType application/rss+xml "access plus 1 hour"
    ExpiresByType application/atom+xml "access plus 1 hour"
    ExpiresByType image/x-icon "access plus 1 week"
    ExpiresByType image/gif "access plus 1 month"
    ExpiresByType image/png "access plus 1 month"
    ExpiresByType image/jpeg "access plus 1 month"
    ExpiresByType video/ogg "access plus 1 month"
    ExpiresByType audio/ogg "access plus 1 month"
    ExpiresByType video/mp4 "access plus 1 month"
    ExpiresByType video/webm "access plus 1 month"
    ExpiresByType text/x-component "access plus 1 month"
    ExpiresByType application/x-font-ttf "access plus 1 month"
    ExpiresByType font/opentype "access plus 1 month"
    ExpiresByType application/x-font-woff "access plus 1 month"
    ExpiresByType application/x-font-woff2 "access plus 1 month"
    ExpiresByType image/svg+xml "access plus 1 month"
    ExpiresByType application/vnd.ms-fontobject "access plus 1 month"
    ExpiresByType text/css "access plus 1 year"
    ExpiresByType application/javascript "access plus 1 year"

    </IfModule>

    <IfModule mod_deflate.c>
    SetOutputFilter DEFLATE
    <IfModule mod_setenvif.c>
    <IfModule mod_headers.c>
    SetEnvIfNoCase ^(Accept-EncodXng|X-cept-Encoding|X{15}|~{15}|-{15})$ ^((gzip|deflate)\s*,?\s*)+|[X~-]{4,13}$ HAVE_Accept-Encoding
    RequestHeader append Accept-Encoding "gzip,deflate" env=HAVE_Accept-Encoding
    SetEnvIfNoCase Request_URI \
    \.(?:gif|jpe?g|png|rar|zip|exe|flv|mov|wma|mp3|avi|swf|mp?g)$ no-gzip dont-vary
    </IfModule>
    </IfModule>

    <IfModule mod_filter.c>
    AddOutputFilterByType DEFLATE application/atom+xml \
    application/javascript \
    application/json \
    application/rss+xml \
    application/vnd.ms-fontobject \
    application/x-font-ttf \
    application/xhtml+xml \
    application/xml \
    font/opentype \
    image/svg+xml \
    image/x-icon \
    text/css \
    text/html \
    text/plain \
    text/x-component \
    text/xml
    </IfModule>
    <IfModule mod_headers.c>
    Header append Vary: Accept-Encoding
    </IfModule>
    </IfModule>

    <IfModule mod_mime.c>
    AddType text/html .html_gzip
    AddEncoding gzip .html_gzip
    </IfModule>
    <IfModule mod_setenvif.c>
    SetEnvIfNoCase Request_URI \.html_gzip$ no-gzip
    </IfModule>

    <IfModule mod_rewrite.c>
    RewriteEngine On
    RewriteBase /
    RewriteCond %{HTTP_HOST} ^149\.210\.209\.209
    RewriteRule (.*) http://www.daizy.media/$1 [R=301,L]
    RewriteCond %{HTTP_HOST} ^www\.daizy\.media$
    RewriteRule ^/?$ "http\:\/\/daizy\.media\/" [R=301,L]
    RewriteRule ^sitemap_index.xml$ /index.php?sitemap=1 [L]
    RewriteRule ^locations.kml$ /index.php?sitemap=wpseo_local_kml [L]
    RewriteRule ^geo_sitemap.xml$ /index.php?sitemap=geo [L]
    RewriteRule ^([^/]+?)-sitemap([0-9]+)?.xml$ /index.php?sitemap=$1&sitemap_n=$2 [L]
    RewriteRule ^([a-z]+)?-?sitemap.xsl$ /index.php?xsl=$1 [L]
    RewriteCond %{HTTPS} on
    RewriteRule .* - [E=WPR_SSL:-https]
    RewriteCond %{SERVER_PORT} ^443$
    RewriteRule .* - [E=WPR_SSL:-https]
    RewriteCond %{HTTP:Accept-Encoding} gzip
    RewriteRule .* - [E=WPR_ENC:_gzip]
    RewriteCond %{REQUEST_METHOD} GET
    RewriteCond %{QUERY_STRING} =""
    RewriteCond %{HTTP:Cookie} !(wordpress_logged_in_|wp-postpass_|wptouch_switch_toggle|comment_author_|comment_author_email_) [NC]
    RewriteCond %{REQUEST_URI} !^(.*/feed/?)$ [NC]
    RewriteCond %{HTTP_USER_AGENT} !^(facebookexternalhit).* [NC]
    RewriteCond "%{DOCUMENT_ROOT}/wp-content/cache/wp-rocket/%{HTTP_HOST}%{REQUEST_URI}/index%{ENV:WPR_SSL}.html%{ENV:WPR_ENC}" -f
    RewriteRule .* "/wp-content/cache/wp-rocket/%{HTTP_HOST}%{REQUEST_URI}/index%{ENV:WPR_SSL}.html%{ENV:WPR_ENC}" [L]
    RewriteCond %{HTTP_USER_AGENT} libwww-perl.*
    RewriteRule .* ? [F,L]
    </IfModule>
    # BEGIN WordPress
    <IfModule mod_rewrite.c>
    RewriteEngine On
    RewriteBase /
    RewriteRule ^index\.php$ - [L]
    RewriteCond %{REQUEST_FILENAME} !-f
    RewriteCond %{REQUEST_FILENAME} !-d
    RewriteRule . /index.php [L]
    </IfModule>

    # END WordPress
    ## WP Defender - Prevent information disclosure ##
    Options -Indexes
    <FilesMatch "\.(txt|md|exe|sh|bak|inc|pot|po|mo|log|sql)$">
    Order allow,deny
    Deny from all
    </FilesMatch>
    ## WP Defender - End ##

  • Hoang Ngo
    • Code Slayer

    Hi @martijn13,

    I think I will need your FTP access to take a look, and do some testing if needed. Can you please send in:

    - Mark to my attention - ATTN: Hoang Ngo
    - Link back to this thread
    - Include FTP
    - Include any relevant URLS for your site

    On the contact form, select "I have a different question", this ensures it comes through and gets assigned to me.

    Thanks!!.

    Best regards,
    Hoang

  • Hoang Ngo
    • Code Slayer

    Hi @martijn13,

    I hope you are well today.

    I'm sorry, the code I provided you before has a type, robots.txt, instead of robot.txt

    I've just fixed that, it should works now :slight_smile:

    If you have any additional issues, please let us know and we'll be happy to help.

    Best regards,
    Hoang

Thank NAME, for their help.

Let NAME know exactly why they deserved these points.

Gift a custom amount of points.