WAF Bot Detection : Support for challenge mode

lib/crowdsec.lua

@@ -8,6 +8,7 @@ local captcha = require "plugins.crowdsec.captcha"
 local flag = require "plugins.crowdsec.flag"
 local utils = require "plugins.crowdsec.utils"
 local ban = require "plugins.crowdsec.ban"
+local challenge = require "plugins.crowdsec.challenge"
 local url = require "plugins.crowdsec.url"
 local metrics = require "plugins.crowdsec.metrics"
 local live = require "plugins.crowdsec.live"
@@ -594,6 +595,11 @@ function csmod.allowIp(ip)
   return true, nil, nil
 end
 
+--- @return boolean: true if the IP did not trigger any WAF rule, false otherwise
+--- @return string: remediation returned by the WAF
+--- @return number: HTTP status code to return to the client
+--- @return table: if the WAF returned a challenge, this table contains the body, headers and cookies to return to the client
+--- @return string: error message if any
 function csmod.AppSecCheck(ip)
   local httpc = http.new()
   httpc:set_timeouts(runtime.conf["APPSEC_CONNECT_TIMEOUT"], runtime.conf["APPSEC_SEND_TIMEOUT"], runtime.conf["APPSEC_PROCESS_TIMEOUT"])
@@ -623,7 +629,7 @@ function csmod.AppSecCheck(ip)
   local body, unreadable_body = get_body()
   if unreadable_body and runtime.conf["APPSEC_DROP_UNREADABLE_BODY"] then
     ngx.log(ngx.WARN, "Dropping request because body is unreadable and APPSEC_DROP_UNREADABLE_BODY is enabled")
-    return false, runtime.conf["FALLBACK_REMEDIATION"], ngx.HTTP_FORBIDDEN, nil
+    return false, runtime.conf["FALLBACK_REMEDIATION"], ngx.HTTP_FORBIDDEN, {}, ""
   end
   if body ~= nil then
     if #body > 0 then
@@ -650,7 +656,7 @@ function csmod.AppSecCheck(ip)
 
   if err ~= nil then
     ngx.log(ngx.ERR, "Fallback because of err: " .. err)
-    return ok, remediation, status_code, err
+    return ok, remediation, status_code, {}, err
   end
 
   if res.status == 200 then
@@ -667,13 +673,21 @@ function csmod.AppSecCheck(ip)
     else
       status_code = ngx.HTTP_FORBIDDEN
     end
+    if remediation == "challenge" then
+      local appsec_response = {
+        body = response.user_body_content,
+        headers = response.user_headers,
+        cookies = response.user_cookies,
+      }
+      return ok, remediation, status_code, appsec_response, ""
+    end
   elseif res.status == 401 then
     ngx.log(ngx.ERR, "Unauthenticated request to APPSEC")
   else
     ngx.log(ngx.ERR, "Bad request to APPSEC (" .. res.status .. "): " .. res.body)
   end
 
-  return ok, remediation, status_code, err
+  return ok, remediation, status_code, {}, ""
 
 end
 
@@ -685,6 +699,7 @@ function csmod.Allow(ip)
   local remediationSource = flag.BOUNCER_SOURCE
   local ret_code = nil
   local remediation = ""
+  local appsec_response = nil
   local ok = true
   local err = ""
   if runtime.conf["ENABLED"] ~= "false" then
@@ -728,7 +743,7 @@ function csmod.Allow(ip)
   -- OR
   -- that user configured the remediation component to always check on the appSec (even if there is a decision for the IP)
   if is_appsec_enabled() and (ok == true or is_always_send_to_appsec())  then
-    local appsecOk, appsecRemediation, status_code, err = csmod.AppSecCheck(ip)
+    local appsecOk, appsecRemediation, status_code, appsec_resp, err = csmod.AppSecCheck(ip)
     if err ~= nil then
       ngx.log(ngx.ERR, "AppSec check: " .. err)
     end
@@ -737,6 +752,7 @@ function csmod.Allow(ip)
       remediationSource = flag.APPSEC_SOURCE
       remediation = appsecRemediation
       ret_code = status_code
+      appsec_response = appsec_resp
     end
   end
 
@@ -749,7 +765,7 @@ function csmod.Allow(ip)
     end
 
     -- if remediation is not supported, fallback
-    if remediation ~= "captcha" and remediation ~= "ban" then
+    if remediation ~= "captcha" and remediation ~= "ban" and remediation ~= "challenge" then
       remediation = runtime.fallback
     end
   end
@@ -824,6 +840,17 @@ function csmod.Allow(ip)
         ban.apply(ret_code)
         return
       end
+      if remediation == "challenge" then
+        if appsec_response ~= nil then
+          ngx.log(ngx.DEBUG, "[Crowdsec] challenge '" .. ip .. "' (by " .. flag.Flags[remediationSource] .. ")")
+          challenge.apply(ret_code, appsec_response.body, appsec_response.headers, appsec_response.cookies)
+          return
+        else
+          ngx.log(ngx.ERR, "[Crowdsec] challenge remediation for '" .. ip .. "' but no response data, falling back to ban")
+          ban.apply(ret_code)
+          return
+        end
+      end
       -- if the remediation is a captcha and captcha is well configured
       if remediation == "captcha" and captcha_ok and ngx.var.uri ~= "/favicon.ico" then
           local previous_uri, flags = ngx.shared.crowdsec_cache:get("captcha_"..ip)

lib/plugins/crowdsec/challenge.lua

@@ -0,0 +1,45 @@
+local M = {
+    _TYPE = 'module',
+    _NAME = 'challenge.funcs',
+    _VERSION = '1.0-0'
+}
+
+--- Serve a challenge response from AppSec.
+-- Sets the HTTP status, response headers, cookies, and body as provided by CrowdSec.
+-- @param status_code number: HTTP status code (typically 200)
+-- @param body string: the HTML body content to serve
+-- @param headers table: map of header name -> list of values, e.g. {["Content-Type"] = {"text/html"}}
+-- @param cookies table: list of Set-Cookie header value strings
+function M.apply(status_code, body, headers, cookies)
+    ngx.status = status_code or ngx.HTTP_OK
+
+    if headers ~= nil then
+        for name, values in pairs(headers) do
+            if type(values) == "table" then
+                if #values == 1 then
+                    ngx.header[name] = values[1]
+                else
+                    ngx.header[name] = values
+                end
+            else
+                ngx.header[name] = values
+            end
+        end
+    end
+
+    if cookies ~= nil and #cookies > 0 then
+        if #cookies == 1 then
+            ngx.header["Set-Cookie"] = cookies[1]
+        else
+            ngx.header["Set-Cookie"] = cookies
+        end
+    end
+
+    if body ~= nil then
+        ngx.print(body)
+    end
+
+    ngx.exit(ngx.status)
+end
+
+return M

t/21_appsec_challenge.t

@@ -0,0 +1,150 @@
+use Test::Nginx::Socket 'no_plan';
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: AppSec 'challenge' remediation serves the challenge page, headers and cookie
+
+--- main_config
+load_module /usr/share/nginx/modules/ndk_http_module.so;
+load_module /usr/share/nginx/modules/ngx_http_lua_module.so;
+
+--- http_config
+
+lua_package_path './lib/?.lua;;';
+lua_shared_dict crowdsec_cache 50m;
+lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+
+
+init_by_lua_block
+{
+        cs = require "crowdsec"
+        local ok, err = cs.init("./t/conf_t/21_appsec_challenge_crowdsec_nginx_bouncer.conf", "crowdsec-nginx-bouncer/v1.0.8")
+        if ok == nil then
+                ngx.log(ngx.ERR, "[Crowdsec] " .. err)
+                error()
+        end
+        ngx.log(ngx.ALERT, "[Crowdsec] Initialisation done")
+}
+
+server {
+    listen 8081;
+
+       location = /v1/decisions {
+            content_by_lua_block {
+                -- no local decision for the IP, so the request reaches AppSec
+                ngx.print('null')
+            }
+       }
+}
+
+server {
+    listen 7422;
+
+       location / {
+            content_by_lua_block {
+                -- AppSec signals a remediation (403) and asks the bouncer to
+                -- serve a challenge page verbatim with a 200 status.
+                ngx.status = 403
+                ngx.print([[{"action":"challenge","http_status":200,"user_body_content":"<html><body>cs-challenge-marker<script>console.log('cs')</script></body></html>","user_headers":{"Content-Type":["text/html"]},"user_cookies":["cs_challenge=abc123; Path=/; HttpOnly"]}]])
+            }
+       }
+}
+
+
+--- config
+
+
+location = /t {
+    set_real_ip_from 127.0.0.1;
+    real_ip_header   X-Forwarded-For;
+    real_ip_recursive on;
+    access_by_lua_block {
+        local cs = require "crowdsec"
+        cs.Allow(ngx.var.remote_addr)
+    }
+    content_by_lua_block {
+        -- must never be reached: the challenge short-circuits the request
+        ngx.say("Hello, world")
+    }
+}
+
+--- raw_request eval
+"GET /t HTTP/1.1\r\nHost: localhost\r\nX-Forwarded-For: 1.1.1.2\r\nConnection: close\r\n\r\n"
+
+--- response_body eval
+"<html><body>cs-challenge-marker<script>console.log('cs')</script></body></html>"
+--- response_headers
+Content-Type: text/html
+Set-Cookie: cs_challenge=abc123; Path=/; HttpOnly
+--- error_code: 200
+
+
+=== TEST 2: AppSec 'challenge' remediation does not leak the protected content
+
+--- main_config
+load_module /usr/share/nginx/modules/ndk_http_module.so;
+load_module /usr/share/nginx/modules/ngx_http_lua_module.so;
+
+--- http_config
+
+lua_package_path './lib/?.lua;;';
+lua_shared_dict crowdsec_cache 50m;
+lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+
+
+init_by_lua_block
+{
+        cs = require "crowdsec"
+        local ok, err = cs.init("./t/conf_t/21_appsec_challenge_crowdsec_nginx_bouncer.conf", "crowdsec-nginx-bouncer/v1.0.8")
+        if ok == nil then
+                ngx.log(ngx.ERR, "[Crowdsec] " .. err)
+                error()
+        end
+        ngx.log(ngx.ALERT, "[Crowdsec] Initialisation done")
+}
+
+server {
+    listen 8081;
+
+       location = /v1/decisions {
+            content_by_lua_block {
+                ngx.print('null')
+            }
+       }
+}
+
+server {
+    listen 7422;
+
+       location / {
+            content_by_lua_block {
+                ngx.status = 403
+                ngx.print([[{"action":"challenge","http_status":200,"user_body_content":"<html><body>cs-challenge-marker</body></html>","user_headers":{"Content-Type":["text/html"]},"user_cookies":["cs_challenge=abc123; Path=/; HttpOnly"]}]])
+            }
+       }
+}
+
+
+--- config
+
+
+location = /t {
+    set_real_ip_from 127.0.0.1;
+    real_ip_header   X-Forwarded-For;
+    real_ip_recursive on;
+    access_by_lua_block {
+        local cs = require "crowdsec"
+        cs.Allow(ngx.var.remote_addr)
+    }
+    content_by_lua_block {
+        ngx.say("protected-origin-content")
+    }
+}
+
+--- raw_request eval
+"GET /t HTTP/1.1\r\nHost: localhost\r\nX-Forwarded-For: 1.1.1.3\r\nConnection: close\r\n\r\n"
+
+--- response_body_unlike: protected-origin-content
+--- error_code: 200

t/conf_t/21_appsec_challenge_crowdsec_nginx_bouncer.conf

@@ -0,0 +1,20 @@
+APPSEC_URL=http://127.0.0.1:7422
+ENABLED=true
+API_URL=http://127.0.0.1:8081
+API_KEY=test_key
+CACHE_EXPIRATION=1
+BOUNCING_ON_TYPE=all
+FALLBACK_REMEDIATION=ban
+REQUEST_TIMEOUT=3000
+UPDATE_FREQUENCY=10
+MODE=live
+EXCLUDE_LOCATION=
+BAN_TEMPLATE_PATH=./ban
+REDIRECT_LOCATION=
+RET_CODE=
+CAPTCHA_PROVIDER=
+SECRET_KEY=
+SITE_KEY=
+CAPTCHA_TEMPLATE_PATH=/var/lib/crowdsec/lua/templates/captcha.html
+CAPTCHA_EXPIRATION=3600
+APPSEC_FAILURE_ACTION=passthrough