Merge pull request #2646 from testssl/fix_feature2098

Feature: Detection STARTTLS throtteling via code 421/SMTP

CHANGELOG.md

@@ -22,8 +22,10 @@
 * BREACH check: list all compression methods and add brotli
 * Test for old winshock vulnerability
 * Test for STARTTLS injection vulnerabilities (SMTP, POP3, IMAP)
-* STARTTLS: XMPP server support, plus new set of OpenSSL-bad binaries
+* STARTTLS: XMPP server support, plus a new set of OpenSSL-bad binaries
+* STARTTLS sieve support, plus again a new set of OpenSSL-bad binaries
 * Several code improvements to STARTTLS, also better detection when no STARTTLS is offered
+* Detect throtteling via STARTTLS smtp
 * Renegotiation checks more reliable against different servers
 * STARTTLS on active directory service support
 * Security fixes: DNS and other input from servers
@@ -41,13 +43,13 @@
 * Added --user-agent argument to support using a custom User Agent
 * Added --overwrite argument to support overwriting output files without warning
 * Headerflag X-XSS-Protection is now labeled as INFO
+* Search for more HTTP security headers on the server
 * Strict parser for HSTS
 * DNS via proxy improvements
 * Client simulation runs in wide mode which is even better readable
 * Added --reqheader to support custom headers in HTTP requests
-* Search for more HTTP security headers on the server
 * Test for support for RFC 8879 certificate compression
-* Deprecating --fast and --ssl-native (warning but still av)
+* Deprecating --fast and --ssl-native (warning only but still av)
 * Compatible to GNU grep 3.8
 * Don't use external pwd command anymore
 * Doesn't hang anymore when there's no local resolver

testssl.sh

@@ -11466,6 +11466,7 @@ starttls_full_read(){
      local end_pattern="$2"
      local starttls_regex="$3"     # optional: pattern we search for in the server's response
      local debug_str="$4"          # optional
+     local problem_pattern="$5"       # optional: used currently only for 421 code
      local starttls_read_data=()
      local one_line=""
      local ret=0
@@ -11486,7 +11487,7 @@ starttls_full_read(){
      while read -r -t $STARTTLS_SLEEP one_line; ret=$?; (exit $ret); do
           debugme tmln_out "S: ${one_line}"
           if [[ $DEBUG -ge 5 ]]; then
-               echo "end_pattern/cont_pattern: ${end_pattern} / ${cont_pattern}"
+               echo "end / cont problem pattern: ${end_pattern} / ${cont_pattern} / ${problem_pattern}"
           fi
           if [[ -n "$starttls_regex" ]]; then
                if [[ ${one_line} =~ $starttls_regex ]]; then
@@ -11494,6 +11495,13 @@ starttls_full_read(){
                     # We don't exit here as the buffer is not empty. So we continue reading but save the status:
                     ret_found=0
                fi
+          elif [[ -n "$problem_pattern" ]]; then
+               if [[ ${one_line} =~ ${problem_pattern} ]]; then
+                    debugme echo "=== matches ${problem_pattern} ==="
+                    IFS="${oldIFS}"
+                    ret_found=4
+                    break
+               fi
           fi
           starttls_read_data+=("${one_line}")
           if [[ ${one_line} =~ ${end_pattern} ]]; then
@@ -11542,6 +11550,7 @@ starttls_smtp_dialog() {
      local greet_str="EHLO testssl.sh"
      local proto="smtp"
      local reSTARTTLS='^250[ -]STARTTLS'
+     local reToofast='^421 '                                # 421 4.7.0 .* Error: too many connections, see #2098
      local starttls="STARTTLS"
      local -i ret=0
 
@@ -11553,13 +11562,14 @@ starttls_smtp_dialog() {
      fi
      debugme echo "=== starting $proto STARTTLS dialog ==="
 
-     starttls_full_read '^220-' '^220 '  ''                 "received server greeting" &&
+     starttls_full_read '^220-' '^220 '  ''                 "received server greeting" "${reToofast}" &&
      starttls_just_send "$greet_str"                        "sent $greet_str" &&
      starttls_full_read '^250-' '^250 '  "${reSTARTTLS}"    "received server capabilities and checked STARTTLS availability" &&
      starttls_just_send "$starttls"                         "initiated STARTTLS" &&
      starttls_full_read '^220-' '^220 '  ''                 "received ack for STARTTLS"
      ret=$?
      debugme echo "=== finished $proto STARTTLS dialog with ${ret} ==="
+     # ret will be 4 if $reToofast matches
      return $ret
 }
 
@@ -11781,9 +11791,13 @@ starttls_telnet_dialog() {
      return $ret
 }
 
-# arg1: fd for socket -- which we don't use yes as it is a hassle (not clear whether it works under every bash version)
+# arg1: fd for socket (which we don't use yet. It's a hassle, not clear whether it works under every bash version
 # arg2: optional: for STARTTLS additional command to be injected
-# returns 6 if opening the socket caused a problem, 1 if STARTTLS handshake failed, 0: all ok
+# return values:
+#    0: all ok
+#    1: STARTTLS handshake failed
+#    4: throtteling on STARTTLS level encountered
+#    6: if opening the socket caused a problem
 #
 fd_socket() {
      local fd="$1"
@@ -11902,6 +11916,9 @@ fd_socket() {
           case $ret in
                0)   return 0 ;;
                3)   fatal "No STARTTLS found in handshake" $ERR_CONNECT ;;
+               4)   ((NR_STARTTLS_FAIL++))
+                    connectivity_problem $NR_STARTTLS_FAIL $MAX_STARTTLS_FAIL "Throtteling detected (STARTTLS server msg 421)" "repeated STARTTLS problems due to throtteling, giving up"
+                    return 4 ;;
                *)   if [[ $ret -eq 2 ]] && [[ -n "$payload" ]]; then
                          # We don't want this handling for STARTTLS injection
                          return 0