Merge branch '3.1dev' into update_clients1

t/32_isHTML_valid.t

@@ -35,7 +35,7 @@ unlink 'tmp.html';
 # Remove the HTML footer
 $edited_html =~ s/\n\<\/pre\>\n\<\/body\>\n\<\/html\>//;
 # Remove any hypertext links for URLs
-$edited_html =~ s/<a href=[0-9A-Za-z ";:=\/\.\?\-]*>//g;
+$edited_html =~ s/<a href=[0-9A-Za-z ";:_&=\/\.\?\-]*>//g;
 $edited_html =~ s/<\/a>//g;
 
 # Replace escaped characters with their original text

t/61_diff_testsslsh.t

@@ -36,7 +36,6 @@ printf "\n%s\n", "Diff unit test IPv4 against \"$uri\"";
 #1 run
 `$prg $check2run $uri 2>&1`;
 
-
 $diff = diff $socket_csv, $master_socket_csv;
 
 $socket_csv=`cat tmp.csv`;
@@ -54,7 +53,6 @@ $master_socket_csv=~ s/censys.io.*\n//g;
 $socket_csv=~ s/HTTP_headerTime.*\n//g;
 $master_socket_csv=~ s/HTTP_headerTime.*\n//g;
 
-
 # Compare the differences to the master file -- and print differences if there were detected.
 #
 cmp_ok($socket_csv, "eq", $master_socket_csv, "Check whether CSV output matches master file from $uri") or

t/baseline_data/default_testssl.csvfile

@@ -66,7 +66,7 @@
 "DH_groups","testssl.sh/81.169.166.184","443","OK","Unknown DH group (2048 bits)","",""
 "HTTP_status_code","testssl.sh/81.169.166.184","443","INFO","200 OK ('/')","",""
 "HTTP_clock_skew","testssl.sh/81.169.166.184","443","INFO","0 seconds from localtime","",""
-"HTTP_headerTime","testssl.sh/81.169.166.184","443","INFO","1652166990","",""
+"HTTP_headerTime","testssl.sh/81.169.166.184","443","INFO","1653487014","",""
 "HSTS_time","testssl.sh/81.169.166.184","443","OK","362 days (=31337000 seconds) > 15552000 seconds","",""
 "HSTS_subdomains","testssl.sh/81.169.166.184","443","INFO","only for this domain","",""
 "HSTS_preload","testssl.sh/81.169.166.184","443","INFO","domain is NOT marked for preloading","",""
@@ -76,7 +76,7 @@
 "cookie_count","testssl.sh/81.169.166.184","443","INFO","0 at '/'","",""
 "X-Frame-Options","testssl.sh/81.169.166.184","443","OK","DENY","",""
 "X-Content-Type-Options","testssl.sh/81.169.166.184","443","OK","nosniff","",""
-"Content-Security-Policy","testssl.sh/81.169.166.184","443","OK","script-src 'unsafe-inline'; style-src 'unsafe-inline' 'self'; default-src 'self' ; child-src 'none'; object-src 'self'; frame-ancestors 'self'; upgrade-insecure-requests","",""
+"Content-Security-Policy","testssl.sh/81.169.166.184","443","OK","script-src 'unsafe-inline'; style-src 'unsafe-inline' 'self'; object-src 'self'; base-uri 'none'; form-action 'none'; img-src 'self' ; default-src 'self'; frame-ancestors 'self'; upgrade-insecure-requests;","",""
 "banner_reverseproxy","testssl.sh/81.169.166.184","443","INFO","--","","CWE-200"
 "heartbleed","testssl.sh/81.169.166.184","443","OK","not vulnerable, no heartbeat extension","CVE-2014-0160","CWE-119"
 "CCS","testssl.sh/81.169.166.184","443","OK","not vulnerable","CVE-2014-0224","CWE-310"
@@ -91,7 +91,7 @@
 "SWEET32","testssl.sh/81.169.166.184","443","OK","not vulnerable","CVE-2016-2183 CVE-2016-6329","CWE-327"
 "FREAK","testssl.sh/81.169.166.184","443","OK","not vulnerable","CVE-2015-0204","CWE-310"
 "DROWN","testssl.sh/81.169.166.184","443","OK","not vulnerable on this host and port","CVE-2016-0800 CVE-2016-0703","CWE-310"
-"DROWN_hint","testssl.sh/81.169.166.184","443","INFO","Make sure you don't use this certificate elsewhere with SSLv2 enabled services, see https://censys.io/ipv4?q=31B44391529821C6A77F3C78B02D716A07F99B8FDB342BF5A78F263C25375968","CVE-2016-0800 CVE-2016-0703","CWE-310"
+"DROWN_hint","testssl.sh/81.169.166.184","443","INFO","Make sure you don't use this certificate elsewhere with SSLv2 enabled services, see https://search.censys.io/search?resource=hosts&virtual_hosts=INCLUDE&q=31B44391529821C6A77F3C78B02D716A07F99B8FDB342BF5A78F263C25375968","CVE-2016-0800 CVE-2016-0703","CWE-310"
 "LOGJAM","testssl.sh/81.169.166.184","443","OK","not vulnerable, no DH EXPORT ciphers,","CVE-2015-4000","CWE-310"
 "LOGJAM-common_primes","testssl.sh/81.169.166.184","443","OK","--","CVE-2015-4000","CWE-310"
 "BEAST_CBC_TLS1","testssl.sh/81.169.166.184","443","MEDIUM","ECDHE-RSA-AES256-SHA ECDHE-RSA-AES128-SHA DHE-RSA-CAMELLIA256-SHA DHE-RSA-CAMELLIA128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA AES256-SHA","CVE-2011-3389","CWE-20"

testssl.sh

@@ -6608,7 +6608,7 @@ sub_session_resumption() {
 
 run_server_preference() {
      local cipher1="" cipher2="" tls13_cipher1="" tls13_cipher2="" default_proto=""
-     local default_cipher=""
+     local default_cipher="" ciph
      local limitedsense="" supported_sslv2_ciphers
      local proto_ossl proto_txt proto_hex cipherlist i
      local -i ret=0 j sclient_success
@@ -6662,27 +6662,28 @@ run_server_preference() {
                esac
           fi
           if [[ $sclient_success -eq 0 ]] ; then
-               cp "$TEMPDIR/$NODEIP.parse_tls_serverhello.txt" $TMPFILE
                cp "$TEMPDIR/$NODEIP.parse_tls_serverhello.txt" "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt"
-               cipher0=$(get_cipher $TMPFILE)
+               cipher0=$(get_cipher "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt")
           fi
      fi
      if [[ $sclient_success -ne 0 ]]; then
-          $OPENSSL s_client $(s_client_options "$STARTTLS $BUGS -connect $NODEIP:$PORT $PROXY $addcmd") </dev/null 2>>$ERRFILE >$TMPFILE
+          $OPENSSL s_client $(s_client_options "$STARTTLS $BUGS -connect $NODEIP:$PORT $PROXY $addcmd") </dev/null 2>>$ERRFILE >"$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt"
-          if sclient_connect_successful $? $TMPFILE; then
+          if sclient_connect_successful $? "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt"; then
-               cipher0=$(get_cipher $TMPFILE)
+               cipher0=$(get_cipher "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt")
                debugme tm_out "0 --> $cipher0\n"
-               cp $TMPFILE "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt"
           else
                # 2 second try with $OPTIMAL_PROTO especially for intolerant IIS6 servers:
-               $OPENSSL s_client $(s_client_options "$STARTTLS $OPTIMAL_PROTO $BUGS -connect $NODEIP:$PORT $PROXY $SNI") </dev/null 2>>$ERRFILE >$TMPFILE
+               $OPENSSL s_client $(s_client_options "$STARTTLS $OPTIMAL_PROTO $BUGS -connect $NODEIP:$PORT $PROXY $SNI") </dev/null 2>>$ERRFILE >"$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt"
-               if ! sclient_connect_successful $? $TMPFILE; then
+               if sclient_connect_successful $? "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt"; then
+                    cipher0=$(get_cipher "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt")
+                    debugme tm_out "0 --> $cipher0\n"
+               else
                     pr_warning "Handshake error!"
                     ret=1
                fi
           fi
      fi
-     [[ $ret -eq 0 ]] && default_proto=$(get_protocol $TMPFILE)
+     [[ $ret -eq 0 ]] && default_proto=$(get_protocol "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt")
      [[ "$default_proto" == TLSv1.0 ]] && default_proto="TLSv1"
      # debugme tm_out " --> $default_proto\n"
 
@@ -6721,41 +6722,50 @@ run_server_preference() {
                pr_warning "no matching cipher in this list found (pls report this): "
                outln "$list_fwd  . "
                fileout "$jsonID" "WARN" "Could not determine server cipher order, no matching cipher in list found (pls report this): $list_fwd"
-               tmpfile_handle ${FUNCNAME[0]}.txt
+               ret=1
-               return 1
                # we assume the problem is with testing here but it could be also the server side
+          else
+               cipher1=$(get_cipher $TMPFILE)               # cipher1 from 1st serverhello
+               debugme tm_out "1 --> $cipher1\n"
+
+               # second client hello with reverse list
+               [[ $DEBUG -ge 4 ]] && echo -e "\n Reverse: ${list_reverse}"
+               $OPENSSL s_client $(s_client_options "$STARTTLS -cipher $list_reverse $BUGS -connect $NODEIP:$PORT $PROXY $addcmd2") </dev/null 2>>$ERRFILE >$TMPFILE
+               # first handshake worked above so no error handling here
+               cipher2=$(get_cipher $TMPFILE)               # cipher2 from 2nd serverhello
+               debugme tm_out "2 --> $cipher2\n"
+
+               [[ $cipher1 == $cipher2 ]] && has_cipher_order=true
           fi
-          cipher1=$(get_cipher $TMPFILE)               # cipher1 from 1st serverhello
-          debugme tm_out "1 --> $cipher1\n"
-
-          # second client hello with reverse list
-          [[ $DEBUG -ge 4 ]] && echo -e "\n Reverse: ${list_reverse}"
-          $OPENSSL s_client $(s_client_options "$STARTTLS -cipher $list_reverse $BUGS -connect $NODEIP:$PORT $PROXY $addcmd2") </dev/null 2>>$ERRFILE >$TMPFILE
-          # first handshake worked above so no error handling here
-          cipher2=$(get_cipher $TMPFILE)               # cipher2 from 2nd serverhello
-          debugme tm_out "2 --> $cipher2\n"
-
-          [[ $cipher1 == $cipher2 ]] && has_cipher_order=true
      fi
      debugme echo "has_cipher_order: $has_cipher_order"
      debugme echo "has_tls13_cipher_order: $has_tls13_cipher_order"
 
      # restore file from above
      [[ "$default_proto" == TLSv1.3 ]] && cp "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt" $TMPFILE
-     cipher1=$(get_cipher $TMPFILE)
+     if [[ "$default_proto" == TLSv1.3 ]] || [[ -n "$cipher2" ]]; then
-     tmpfile_handle ${FUNCNAME[0]}.txt
+          cipher1=$(get_cipher $TMPFILE)
+          tmpfile_handle ${FUNCNAME[0]}.txt
+     fi
 
      # Sanity check: Handshake with no ciphers and one with forward list didn't overlap
-     if [[ "$cipher0" != $cipher1 ]]; then
+     if [[ $ret -eq 0 ]] && [[ "$cipher0" != $cipher1 ]]; then
           limitedsense=" (matching cipher in list missing)"
      fi
 
-     if [[ "$DISPLAY_CIPHERNAMES" =~ openssl ]] && ( [[ "$cipher1" == TLS_* ]] || [[ "$cipher1" == SSL_* ]] ); then
+     if [[ -n "$cipher1" ]]; then
-          default_cipher="$(rfc2openssl "$cipher1")"
+          ciph="$cipher1"
-     elif [[ "$DISPLAY_CIPHERNAMES" =~ rfc ]] && [[ "$cipher1" != TLS_* ]] && [[ "$cipher1" != SSL_* ]]; then
+     else
-          default_cipher="$(openssl2rfc "$cipher1")"
+          ciph="$cipher0"
+          cp "$TEMPDIR/$NODEIP.parse_tls13_serverhello.txt" $TMPFILE
+          tmpfile_handle ${FUNCNAME[0]}.txt
      fi
-     [[ -z "$default_cipher" ]] && default_cipher="$cipher1"
+     if [[ "$DISPLAY_CIPHERNAMES" =~ openssl ]] && ( [[ "$ciph" == TLS_* ]] || [[ "$ciph" == SSL_* ]] ); then
+          default_cipher="$(rfc2openssl "$ciph")"
+     elif [[ "$DISPLAY_CIPHERNAMES" =~ rfc ]] && [[ "$ciph" != TLS_* ]] && [[ "$ciph" != SSL_* ]]; then
+          default_cipher="$(openssl2rfc "$ciph")"
+     fi
+     [[ -z "$default_cipher" ]] && default_cipher="$ciph"
 
      "$FAST" && using_sockets=false
      [[ $TLS_NR_CIPHERS == 0 ]] && using_sockets=false
@@ -6778,6 +6788,8 @@ run_server_preference() {
                     outln " (listed by strength)"
                elif [[ $proto_ossl == tls1_3 ]]; then
                     outln " (no server order, thus listed by strength)"
+               elif [[ -z "$cipher2" ]]; then
+                    outln " (listed by strength)"
                else
                     prln_svrty_high " (no server order, thus listed by strength)"
                fi
@@ -6794,6 +6806,8 @@ run_server_preference() {
           out "no (TLS 1.3 only)"
           limitedsense=" (limited sense as client will pick)"
           fileout "$jsonID" "INFO" "not a cipher order for TLS 1.3 configured"
+     elif ! "$TLS13_ONLY" && [[ -z "$cipher2" ]]; then
+          pr_warning "unable to determine"
      elif ! "$has_cipher_order" && ! "$has_tls13_cipher_order"; then
           # server used the different ends (ciphers) from the client hello
           pr_svrty_high "no (NOT ok)"
@@ -6895,11 +6909,11 @@ run_server_preference() {
           *)   fileout "$jsonID" "INFO" "$default_cipher$(read_dhbits_from_file "$TEMPDIR/$NODEIP.run_server_preference.txt" "string") $limitedsense"
                ;;
      esac
-     read_dhbits_from_file "$TEMPDIR/$NODEIP.run_server_preference.txt"
+     [[ -n "$default_cipher" ]] && read_dhbits_from_file "$TEMPDIR/$NODEIP.run_server_preference.txt"
 
      if [[ "$cipher0" != $cipher1 ]]; then
           pr_warning " -- inconclusive test, matching cipher in list missing"
-          outln ", better see below"
+          outln ", better see above"
           #FIXME: This is ugly but the best we can do before rewrite this section
      else
           outln "$limitedsense"
@@ -16705,6 +16719,7 @@ run_crime() {
 
      [[ $VULN_COUNT -le $VULN_THRESHLD ]] && outln && pr_headlineln " Testing for CRIME vulnerability " && outln
      pr_bold " CRIME, TLS " ; out "($cve)                "
+     jsonID="CRIME_TLS"
 
      if "$TLS13_ONLY"; then
           pr_svrty_best "not vulnerable (OK)"
@@ -16717,7 +16732,7 @@ run_crime() {
      if ! "$HAS_ZLIB"; then
           if "$SSL_NATIVE"; then
                prln_local_problem "$OPENSSL lacks zlib support"
-               fileout "CRIME_TLS" "WARN" "CRIME, TLS: Not tested. $OPENSSL lacks zlib support" "$cve" "$cwe"
+               fileout "$jsonID" "WARN" "CRIME, TLS: Not tested. $OPENSSL lacks zlib support" "$cve" "$cwe"
                return 1
           else
                tls_sockets "03" "$TLS12_CIPHER" "" "" "true"
@@ -16735,23 +16750,23 @@ run_crime() {
 
      if [[ $sclient_success -ne 0 ]]; then
           pr_warning "test failed (couldn't connect)"
-          fileout "CRIME_TLS" "WARN" "Check failed, couldn't connect" "$cve" "$cwe"
+          fileout "$jsonID" "WARN" "Check failed, couldn't connect" "$cve" "$cwe"
           ret=1
      elif grep -a Compression $TMPFILE | grep -aq NONE >/dev/null; then
           pr_svrty_good "not vulnerable (OK)"
           if [[ $SERVICE != HTTP ]] && [[ "$CLIENT_AUTH" != required ]];  then
                out " (not using HTTP anyway)"
-               fileout "CRIME_TLS" "OK" "not vulnerable (not using HTTP anyway)" "$cve" "$cwe"
+               fileout "$jsonID" "OK" "not vulnerable (not using HTTP anyway)" "$cve" "$cwe"
           else
-               fileout "CRIME_TLS" "OK" "not vulnerable" "$cve" "$cwe"
+               fileout "$jsonID" "OK" "not vulnerable" "$cve" "$cwe"
           fi
      else
           if [[ $SERVICE == HTTP ]] || [[ "$CLIENT_AUTH" == required ]]; then
                pr_svrty_high "VULNERABLE (NOT ok)"
-               fileout "CRIME_TLS" "HIGH" "VULNERABLE" "$cve" "$cwe" "$hint"
+               fileout "$jsonID" "HIGH" "VULNERABLE" "$cve" "$cwe" "$hint"
           else
                pr_svrty_medium "VULNERABLE but not using HTTP: probably no exploit known"
-               fileout "CRIME_TLS" "MEDIUM" "VULNERABLE, but not using HTTP. Probably no exploit known" "$cve" "$cwe" "$hint"
+               fileout "$jsonID" "MEDIUM" "VULNERABLE, but not using HTTP. Probably no exploit known" "$cve" "$cwe" "$hint"
                # not clear whether a protocol != HTTP offers the ability to repeatedly modify the input
                # which is done e.g. via javascript in the context of HTTP
           fi
@@ -17802,6 +17817,7 @@ run_drown() {
      local cwe="CWE-310"
      local hint=""
      local jsonID="DROWN"
+     local censys_host_url="https://search.censys.io/search?resource=hosts&virtual_hosts=INCLUDE"
 
      if [[ $VULN_COUNT -le $VULN_THRESHLD ]]; then
           outln
@@ -17827,6 +17843,7 @@ run_drown() {
           return 1
      fi
 
+     censys_host_url="$censys_host_url&q=$cert_fingerprint_sha2"
      if [[ $(has_server_protocol ssl2) -ne 1 ]]; then
           sslv2_sockets
      else
@@ -17849,26 +17866,26 @@ run_drown() {
                     nr_ciphers_detected=$((V2_HELLO_CIPHERSPEC_LENGTH / 3))
                     if [[ 0 -eq "$nr_ciphers_detected" ]]; then
                          prln_svrty_high "CVE-2015-3197: SSLv2 supported but couldn't detect a cipher (NOT ok)";
-                         fileout "$jsonID" "HIGH" "SSLv2 offered, but could not detect a cipher. Make sure you don't use this certificate elsewhere, see https://censys.io/ipv4?q=$cert_fingerprint_sha2" "$cve CVE-2015-3197" "$cwe" "$hint"
+                         fileout "$jsonID" "HIGH" "SSLv2 offered, but could not detect a cipher. Make sure you don't use this certificate elsewhere, see $censys_host_url" "$cve CVE-2015-3197" "$cwe" "$hint"
                     else
                          prln_svrty_critical  "VULNERABLE (NOT ok), SSLv2 offered with $nr_ciphers_detected ciphers";
-                         fileout "$jsonID" "CRITICAL" "VULNERABLE, SSLv2 offered with $nr_ciphers_detected ciphers. Make sure you don't use this certificate elsewhere, see https://censys.io/ipv4?q=$cert_fingerprint_sha2" "$cve" "$cwe" "$hint"
+                         fileout "$jsonID" "CRITICAL" "VULNERABLE, SSLv2 offered with $nr_ciphers_detected ciphers. Make sure you don't use this certificate elsewhere, see $censys_host_url" "$cve" "$cwe" "$hint"
                          set_grade_cap "F" "Vulnerable to DROWN"
                     fi
                     outln "$spaces Make sure you don't use this certificate elsewhere, see:"
                     out "$spaces "
-                    pr_url "https://censys.io/ipv4?q=$cert_fingerprint_sha2"
+                    pr_url "$censys_host_url"
                     outln
                fi
                ;;
           *)   prln_svrty_best "not vulnerable on this host and port (OK)"
                fileout "$jsonID" "OK" "not vulnerable on this host and port" "$cve" "$cwe"
                if [[ -n "$cert_fingerprint_sha2" ]]; then
-                    outln "$spaces make sure you don't use this certificate elsewhere with SSLv2 enabled services"
+                    outln "$spaces make sure you don't use this certificate elsewhere with SSLv2 enabled services, see"
                     out "$spaces "
-                    pr_url "https://censys.io/ipv4?q=$cert_fingerprint_sha2"
+                    pr_url "$censys_host_url"
-                    outln " could help you to find out"
+                    outln
-                    fileout "${jsonID}_hint" "INFO" "Make sure you don't use this certificate elsewhere with SSLv2 enabled services, see https://censys.io/ipv4?q=$cert_fingerprint_sha2" "$cve" "$cwe"
+                    fileout "${jsonID}_hint" "INFO" "Make sure you don't use this certificate elsewhere with SSLv2 enabled services, see $censys_host_url" "$cve" "$cwe"
                else
                     outln "$spaces no RSA certificate, thus certificate can't be used with SSLv2 elsewhere"
                     fileout "${jsonID}_hint" "INFO" "no RSA certificate, can't be used with SSLv2 elsewhere" "$cve" "$cwe"
@@ -19554,7 +19571,7 @@ find_openssl_binary() {
      local openssl_location cwd=""
      local ossl_wo_dev_info
      local curve
-     local -a curves_ossl=("sect163k1" "sect163r1" "sect163r2" "sect193r1" "sect193r2" "sect233k1" "sect233r1" "sect239k1" "sect283k1" "sect283r1" "sect409k1" "sect409r1" "sect571k1" "sect571r1" "secp160k1" "secp160r1" "secp160r2" "secp192k1" "prime192v1" "secp224k1" "secp224r1" "secp256k1" "prime256v1" "secp384r1" "secp521r1" "brainpoolP256r1" "brainpoolP384r1" "brainpoolP512r1" "X25519" "X448")
+     local -a curves_ossl=("sect163k1" "sect163r1" "sect163r2" "sect193r1" "sect193r2" "sect233k1" "sect233r1" "sect239k1" "sect283k1" "sect283r1" "sect409k1" "sect409r1" "sect571k1" "sect571r1" "secp160k1" "secp160r1" "secp160r2" "secp192k1" "prime192v1" "secp224k1" "secp224r1" "secp256k1" "prime256v1" "secp384r1" "secp521r1" "brainpoolP256r1" "brainpoolP384r1" "brainpoolP512r1" "X25519" "X448" "ffdhe2048" "ffdhe3072" "ffdhe4096" "ffdhe6144" "ffdhe8192")
 
      # 0. check environment variable whether it's executable
      if [[ -n "$OPENSSL" ]] && [[ ! -x "$OPENSSL" ]]; then