Merge branch 'ocsp_crl_final' into 2.9dev

and bump version to 3.0rc1

testssl.sh

@@ -116,7 +116,7 @@ trap "child_error" USR1
 
 ########### Internal definitions
 #
-declare -r VERSION="3.0beta"
+declare -r VERSION="3.0rc1"
 declare -r SWCONTACT="dirk aet testssl dot sh"
 egrep -q "dev|rc|beta" <<< "$VERSION" && \
      SWURL="https://testssl.sh/dev/" ||
@@ -225,6 +225,7 @@ OFFENSIVE=${OFFENSIVE:-true}            # do you want to include offensive vulne
 ########### Tuning vars which cannot be set by a cmd line switch. Use instead e.g "HEADER_MAXSLEEP=10 ./testssl.sh <your_args_here>"
 #
 EXPERIMENTAL=${EXPERIMENTAL:-false}
+PROXY_WAIT=${PROXY_WAIT:-20}            # waiting at max 20 seconds for socket reply through proxy
 HEADER_MAXSLEEP=${HEADER_MAXSLEEP:-5}   # we wait this long before killing the process to retrieve a service banner / http header
 MAX_SOCKET_FAIL=${MAX_SOCKET_FAIL:-2}   # If this many failures for TCP socket connects are reached we terminate
 MAX_OSSL_FAIL=${MAX_OSSL_FAIL:-2}       # If this many failures for s_client connects are reached we terminate
@@ -242,8 +243,8 @@ HPKP_MIN=${HPKP_MIN:-30}                # >=30 days should be ok for HPKP_MIN, p
      HPKP_MIN=$((HPKP_MIN * 86400))     # correct to seconds
 DAYS2WARN1=${DAYS2WARN1:-60}            # days to warn before cert expires, threshold 1
 DAYS2WARN2=${DAYS2WARN2:-30}            # days to warn before cert expires, threshold 2
-VULN_THRESHLD=${VULN_THRESHLD:-1}       # if vulnerabilities to check >$VULN_THRESHLD we DON'T show a separate header line in the output each vuln. check
+pVULN_THRESHLD=${VULN_THRESHLD:-1}       # if vulnerabilities to check >$VULN_THRESHLD we DON'T show a separate header line in the output each vuln. check
-DNS_VIA_PROXY=${DNS_VIA_PROXY:-false}   # don't do DNS lookups via proxy. --ip=proxy reverses this
+DNS_VIA_PROXY=${DNS_VIA_PROXY:-true}    # do DNS lookups via proxy. --ip=proxy reverses this
 UNBRACKTD_IPV6=${UNBRACKTD_IPV6:-false} # some versions of OpenSSL (like Gentoo) don't support [bracketed] IPv6 addresses
 NO_ENGINE=${NO_ENGINE:-false}           # if there are problems finding the (external) openssl engine set this to true
 declare -r CLIENT_MIN_PFS=5             # number of ciphers needed to run a test for PFS
@@ -1398,28 +1399,67 @@ filter_input() {
 # Dl's any URL (arg1) via HTTP 1.1 GET from port 80, arg2: file to store http body.
 # Proxy is not honored yet (see cmd line switches) -- except when using curl or wget.
 # There the environment variable is used automatically
-#
+# Currently it is being used by check_revocation_crl() only.
 http_get() {
      local proto z
      local node="" query=""
      local dl="$2"
      local useragent="$UA_STD"
+     local proxy_arg=""
+     local jsonID="http_get"
 
      "$SNEAKY" && useragent="$UA_SNEAKY"
 
-     # automatically handles proxy vars via ENV
      if type -p curl &>/dev/null; then
-          curl -s -A $''"$useragent"'' -o $dl "$1"
+          if [[ -z "$PROXY" ]]; then
+               curl -s --noproxy '*' -A $''"$useragent"'' -o $dl "$1"
+          else
+               # for the sake of simplicity assume the proxy is using http
+               curl -s -x $PROXYIP:$PROXYPORT -A $''"$useragent"'' -o $dl "$1"
+          fi
           return $?
      elif type -p wget &>/dev/null; then
-          wget -q -U $''"$useragent"'' -O $dl "$1"
+          # wget has no proxy command line. We need to use http_proxy instead. And for the sake of simplicity
+          # assume the GET protocol we query is using http -- http_proxy is the $ENV not for the connection TO
+          # the proxy, but for the protocol we query THROUGH the proxy
+          if [[ -z "$PROXY" ]]; then
+               wget --no-proxy -q -U $''"$useragent"'' -O $dl "$1"
+          else
+               if [[ -z "$http_proxy" ]]; then
+                    http_proxy=http://$PROXYIP:$PROXYPORT wget -q -U $''"$useragent"'' -O $dl "$1"
+               else
+                    wget -q -U $''"$useragent"'' -O $dl "$1"
+               fi
+          fi
           return $?
      else
-          # Worst option: slower and hiccups with chunked transfers.  Workround for the latter is HTTP/1.0
+          # Worst option: slower and hiccups with chunked transfers. Workround for the
+          # latter is using HTTP/1.0. We do not support https here, yet.
+          # First the URL will be split
           IFS=/ read -r proto z node query <<< "$1"
-          exec 33<>/dev/tcp/$node/80
+          proto=${proto%:}
-          printf -- "%b" "GET /$query HTTP/1.0\r\nUser-Agent: $useragent\r\nHost: $node\r\nAccept: */*\r\n\r\n" >&33
+          if [[ "$proto" != http ]]; then
-          # strip HTTP header (
+               pr_warning "protocol $proto not supported yet"
+               fileout "$jsonID" "DEBUG" "protocol $proto not supported yet"
+               return 6
+          fi
+          if [[ -n $PROXY ]]; then
+               # PROXYNODE works better than PROXYIP on modern versions of squid. \
+               # We don't reuse the code in fd_socket() as there's initial CONNECT which makes problems
+               if ! exec 33<> /dev/tcp/${PROXYNODE}/${PROXYPORT}; then
+                    outln
+                    pr_warning "$PROG_NAME: unable to open a socket to proxy $PROXYNODE:$PROXYPORT"
+                    fileout "$jsonID" "DEBUG" "$PROG_NAME: unable to open a socket to proxy $PROXYNODE:$PROXYPORT"
+                    return 6
+               else
+                    printf -- "%b" "GET $proto://$node/$query HTTP/1.0\r\nUser-Agent: $useragent\r\nHost: $node\r\nAccept: */*\r\n\r\n" >&33
+               fi
+          else
+               IFS=/ read -r proto z node query <<< "$1"
+               exec 33<>/dev/tcp/$node/80
+               printf -- "%b" "GET /$query HTTP/1.0\r\nUser-Agent: $useragent\r\nHost: $node\r\nAccept: */*\r\n\r\n" >&33
+          fi
+          # Strip HTTP header. When in Debug Mode we leave the raw data in place
           if [[ $DEBUG -ge 1 ]]; then
                cat <&33 >${dl}.raw
                cat ${dl}.raw | sed '1,/^[[:space:]]*$/d' >${dl}
@@ -1427,6 +1467,7 @@ http_get() {
                cat <&33 | sed '1,/^[[:space:]]*$/d' >${dl}
           fi
           exec 33<&-
+          exec 33>&-
           [[ -s "$dl" ]] && return 0 || return 1
      fi
 }
@@ -1439,6 +1480,7 @@ ldap_get() {
      local jsonID="$3"
 
      if type -p curl &>/dev/null; then
+          # proxy handling?
           ldif="$(curl -s "$crl")"
           [[ $? -eq 0 ]] || return 1
           awk '/certificateRevocationList/ { print $2 }' <<< "$ldif" | $OPENSSL base64 -d -A -out "$tmpfile" 2>/dev/null
@@ -1474,6 +1516,7 @@ check_revocation_crl() {
      if [[ $success -eq 2 ]]; then
           return 0
      elif [[ $success -ne 0 ]]; then
+          out ", "
           pr_warning "retrieval of \"$crl\" failed"
           fileout "$jsonID" "WARN" "CRL retrieval from $crl failed"
           return 1
@@ -1529,9 +1572,9 @@ check_revocation_ocsp() {
      grep -q "\-\-\-\-\-BEGIN CERTIFICATE\-\-\-\-\-" $TEMPDIR/intermediatecerts.pem || return 0
      tmpfile=$TEMPDIR/${NODE}-${NODEIP}.${uri##*\/} || exit $ERR_FCREATE
      if [[ -n "$stapled_response" ]]; then
-          > "$TEMPDIR/stabled_ocsp_response.dd"
+          > "$TEMPDIR/stapled_ocsp_response.dd"
-          asciihex_to_binary_file "$stapled_response" "$TEMPDIR/stabled_ocsp_response.dd"
+          asciihex_to_binary_file "$stapled_response" "$TEMPDIR/stapled_ocsp_response.dd"
-          $OPENSSL ocsp -no_nonce -respin "$TEMPDIR/stabled_ocsp_response.dd" \
+          $OPENSSL ocsp -no_nonce -respin "$TEMPDIR/stapled_ocsp_response.dd" \
                -issuer $TEMPDIR/hostcert_issuer.pem -verify_other $TEMPDIR/intermediatecerts.pem \
                -CAfile <(cat $ADDITIONAL_CA_FILES "$GOOD_CA_BUNDLE") -cert $HOSTCERT -text &> "$tmpfile"
      else
@@ -1732,26 +1775,26 @@ s_client_options() {
      # any TLSv1.3 ciphers, then the command will always fail. So, if $OPENSSL supports
      # TLSv1.3 and a cipher list is provided, but no protocol is specified, then add
      # -no_tls1_3 if no TLSv1.3 ciphers are provided.
-     if "$HAS_TLS13" && [[ "$ciphers" != "notpresent" ]] && \
+     if "$HAS_TLS13" && [[ "$ciphers" != notpresent ]] && \
-          ( [[ "$tls13_ciphers" == "notpresent" ]] || [[ -z "$tls13_ciphers" ]] ) && \
+          ( [[ "$tls13_ciphers" == notpresent ]] || [[ -z "$tls13_ciphers" ]] ) && \
           [[ ! " $options " =~ \ -ssl[2|3]\  ]] && \
           [[ ! " $options " =~ \ -tls1\  ]] && \
           [[ ! " $options " =~ \ -tls1_[1|2|3]\  ]]; then
           options+=" -no_tls1_3"
      fi
 
-     if [[ "$ciphers" != "notpresent" ]] || [[ "$tls13_ciphers" != "notpresent" ]]; then
+     if [[ "$ciphers" != notpresent ]] || [[ "$tls13_ciphers" != notpresent ]]; then
           if ! "$HAS_CIPHERSUITES"; then
-               [[ "$ciphers" == "notpresent" ]] && ciphers=""
+               [[ "$ciphers" == notpresent ]] && ciphers=""
-               [[ "$tls13_ciphers" == "notpresent" ]] && tls13_ciphers=""
+               [[ "$tls13_ciphers" == notpresent ]] && tls13_ciphers=""
                [[ -n "$ciphers" ]] && [[ -n "$tls13_ciphers" ]] && ciphers=":$ciphers"
                ciphers="$tls13_ciphers$ciphers"
                options+=" -cipher $ciphers"
           else
-               if [[ "$ciphers" != "notpresent" ]] && [[ -n "$ciphers" ]]; then
+               if [[ "$ciphers" != notpresent ]] && [[ -n "$ciphers" ]]; then
                     options+=" -cipher $ciphers"
                fi
-               if [[ "$tls13_ciphers" != "notpresent" ]] && [[ -n "$tls13_ciphers" ]]; then
+               if [[ "$tls13_ciphers" != notpresent ]] && [[ -n "$tls13_ciphers" ]]; then
                     options+=" -ciphersuites $tls13_ciphers"
                fi
           fi
@@ -8915,34 +8958,41 @@ starttls_mysql_dialog() {
      return $ret
 }
 
-# arg for a fd doesn't work here
+# arg1: fd for socket -- which we don't use as it is a hassle and it is not clear whether it works under every bash version
+#
 fd_socket() {
      local jabber=""
      local proyxline=""
      local nodeip="$(tr -d '[]' <<< $NODEIP)"          # sockets do not need the square brackets we have of IPv6 addresses
                                                        # we just need do it here, that's all!
      if [[ -n "$PROXY" ]]; then
-          if ! exec 5<> /dev/tcp/${PROXYIP}/${PROXYPORT}; then
+          # PROXYNODE works better than PROXYIP on modern versions of squid
+          if ! exec 5<> /dev/tcp/${PROXYNODE}/${PROXYPORT}; then
                outln
-               pr_warning "$PROG_NAME: unable to open a socket to proxy $PROXYIP:$PROXYPORT"
+               pr_warning "$PROG_NAME: unable to open a socket to proxy $PROXYNODE:$PROXYPORT"
                return 6
           fi
           if "$DNS_VIA_PROXY"; then
-               echo -e "CONNECT $NODE:$PORT HTTP/1.0\n" >&5
+               printf -- "%b" "CONNECT $NODE:$PORT HTTP/1.0\n\n" >&5
           else
-               echo -e "CONNECT $nodeip:$PORT HTTP/1.0\n" >&5
+               printf -- "%b" "CONNECT $nodeip:$PORT HTTP/1.0\n\n" >&5
           fi
-          while true ; do
+          while true; do
-               read -r proyxline <&5
+               read -t $PROXY_WAIT -r proyxline <&5
-               if [[ "${proyxline%/*}" == "HTTP" ]]; then
+               if [[ $? -ge 128 ]]; then
+                    pr_warning "Proxy timed out. Unable to CONNECT via proxy. "
+                    close_socket
+                    return 6
+               elif [[ "${proyxline%/*}" == HTTP ]]; then
                     proyxline=${proyxline#* }
-                    if [[ "${proyxline%% *}" != "200" ]]; then
+                    if [[ "${proyxline%% *}" != 200 ]]; then
                          pr_warning "Unable to CONNECT via proxy. "
                          [[ "$PORT" != 443 ]] && prln_warning "Check whether your proxy supports port $PORT and the underlying protocol."
+                         close_socket
                          return 6
                     fi
                fi
-               if [[ "$proyxline" == $'\r' ]]; then
+               if [[ "$proyxline" == $'\r' ]] || [[ -z "$proyxline" ]] ; then
                     break
                fi
           done
@@ -15364,7 +15414,7 @@ tuning / connect options (most also can be preset via environment variables):
      --assume-http                 if protocol check fails it assumes HTTP protocol and enforces HTTP checks
      --ssl-native                  fallback to checks with OpenSSL where sockets are normally used
      --openssl <PATH>              use this openssl binary (default: look in \$PATH, \$RUN_DIR of $PROG_NAME)
-     --proxy <host:port|auto>      connect via the specified HTTP proxy, auto: autodetermination from \$env (\$http(s)_proxy)
+     --proxy <host:port|auto>      (experimental) proxy connects via <host:port>, auto: values from \$env (\$http(s)_proxy)
      -6                            also use IPv6. Works only with supporting OpenSSL version and IPv6 connectivity
      --ip <ip>                     a) tests the supplied <ip> v4 or v6 address instead of resolving host(s) in URI
                                    b) arg "one" means: just test the first DNS returns (useful for multiple IPs)
@@ -15455,7 +15505,6 @@ OSSL_VER_PLATFORM: $OSSL_VER_PLATFORM
 
 OPENSSL_NR_CIPHERS: $OPENSSL_NR_CIPHERS
 OPENSSL_CONF: $OPENSSL_CONF
-
 OSSL_SUPPORTED_CURVES: $OSSL_SUPPORTED_CURVES
 
 HAS_IPv6: $HAS_IPv6
@@ -15842,6 +15891,11 @@ get_a_record() {
      local saved_openssl_conf="$OPENSSL_CONF"
 
      [[ "$NODNS" == none ]] && return 0      # if no DNS lookup was instructed, leave here
+     if [[ "$1" == localhost ]]; then
+          # This is a bit ugly but prevents from doing DNS lookups which could fail
+          echo 127.0.0.1
+          return 0
+     fi
      OPENSSL_CONF=""                         # see https://github.com/drwetter/testssl.sh/issues/134
      check_resolver_bins
      if [[ "$NODE" == *.local ]]; then
@@ -16128,7 +16182,7 @@ check_proxy() {
           fi
           # strip off http/https part if supplied:
           PROXY="${PROXY/http\:\/\//}"
-          PROXY="${PROXY/https\:\/\//}"
+          PROXY="${PROXY/https\:\/\//}"      # this shouldn't be needed
           PROXYNODE="${PROXY%:*}"
           PROXYPORT="${PROXY#*:}"
           is_number "$PROXYPORT" || fatal "Proxy port cannot be determined from \"$PROXY\"" $ERR_CMDLINE
@@ -16278,7 +16332,7 @@ determine_service() {
      local ua
      local protocol
 
-     if ! fd_socket; then          # check if we can connect to $NODEIP:$PORT
+     if ! fd_socket 5; then          # check if we can connect to $NODEIP:$PORT
           if [[ -n "$PROXY" ]]; then
                fatal "You're sure $PROXYNODE:$PROXYPORT allows tunneling here? Can't connect to \"$NODEIP:$PORT\"" $ERR_CONNECT
           else