diff --git a/deployment/.htaccess b/deployment/.htaccess new file mode 100644 index 0000000..9f48464 --- /dev/null +++ b/deployment/.htaccess @@ -0,0 +1,18 @@ +# Disallow known bad bots +SetEnvIfNoCase User-Agent "MJ12bot" bad_bot +SetEnvIfNoCase User-Agent "AhrefsBot" bad_bot +SetEnvIfNoCase User-Agent "SemrushBot" bad_bot +Deny from env=bad_bot + +# Rate limit using mod_ratelimit (if available) + + SetOutputFilter RATE_LIMIT + SetEnv rate-limit 400 + + +# Prevent crawlers from indexing results pages + + RewriteEngine On + RewriteCond %{REQUEST_URI} ^/results/thunderball + RewriteRule .* - [F] + \ No newline at end of file diff --git a/deployment/robots.txt b/deployment/robots.txt new file mode 100644 index 0000000..626ebc2 --- /dev/null +++ b/deployment/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /results/ +Disallow: /tickets/ \ No newline at end of file