18 lines
513 B
ApacheConf
18 lines
513 B
ApacheConf
# Disallow known bad bots
|
|
SetEnvIfNoCase User-Agent "MJ12bot" bad_bot
|
|
SetEnvIfNoCase User-Agent "AhrefsBot" bad_bot
|
|
SetEnvIfNoCase User-Agent "SemrushBot" bad_bot
|
|
Deny from env=bad_bot
|
|
|
|
# Rate limit using mod_ratelimit (if available)
|
|
<IfModule mod_ratelimit.c>
|
|
SetOutputFilter RATE_LIMIT
|
|
SetEnv rate-limit 400
|
|
</IfModule>
|
|
|
|
# Prevent crawlers from indexing results pages
|
|
<IfModule mod_rewrite.c>
|
|
RewriteEngine On
|
|
RewriteCond %{REQUEST_URI} ^/results/thunderball
|
|
RewriteRule .* - [F]
|
|
</IfModule> |