Merge branch 'master' into no_version_tolerance_test

This commit is contained in:
David Cooper
2016-10-27 16:51:50 -04:00
10 changed files with 1932 additions and 1378 deletions

View File

@ -83,7 +83,7 @@ readonly PS4='${LINENO}> ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
# make sure that temporary files are cleaned up after use in ANY case
trap "cleanup" QUIT EXIT
readonly VERSION="2.8rc2"
readonly VERSION="2.8rc3"
readonly SWCONTACT="dirk aet testssl dot sh"
egrep -q "dev|rc" <<< "$VERSION" && \
SWURL="https://testssl.sh/dev/" ||
@ -143,7 +143,7 @@ SHOW_SIGALGO=${SHOW_SIGALGO:-false} # "secret" switch whether testssl.sh sho
SNEAKY=${SNEAKY:-false} # is the referer and useragent we leave behind just usual?
QUIET=${QUIET:-false} # don't output the banner. By doing this yiu acknowledge usage term appearing in the banner
SSL_NATIVE=${SSL_NATIVE:-false} # we do per default bash sockets where possible "true": switch back to "openssl native"
ASSUMING_HTTP=${ASSUMING_HTTP:-false} # in seldom cases (WAF, old servers, grumpy SSL) service detection fails. "True" enforces HTTP checks
ASSUME_HTTP=${ASSUME_HTTP:-false} # in seldom cases (WAF, old servers, grumpy SSL) service detection fails. "True" enforces HTTP checks
BUGS=${BUGS:-""} # -bugs option from openssl, needed for some BIG IP F5
DEBUG=${DEBUG:-0} # 1: normal putput the files in /tmp/ are kept for further debugging purposes
# 2: list more what's going on , also lists some errors of connections
@ -251,7 +251,6 @@ TLS_NOW=""
NOW_TIME=""
HTTP_TIME=""
GET_REQ11=""
HEAD_REQ10=""
readonly UA_STD="TLS tester from $SWURL"
readonly UA_SNEAKY="Mozilla/5.0 (X11; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0"
FIRST_FINDING=true # Is this the first finding we are outputting to file?
@ -377,6 +376,7 @@ pr_off() { [[ "$COLOR" -ne 0 ]] && out "\033[m"; }
pr_bold() { [[ "$COLOR" -ne 0 ]] && out "\033[1m$1" || out "$1"; pr_off; }
pr_boldln() { pr_bold "$1" ; outln; }
pr_italic() { [[ "$COLOR" -ne 0 ]] && out "\033[3m$1" || out "$1"; pr_off; }
pr_italicln() { pr_italic "$1" ; outln; }
pr_underline() { [[ "$COLOR" -ne 0 ]] && out "\033[4m$1" || out "$1"; pr_off; }
pr_reverse() { [[ "$COLOR" -ne 0 ]] && out "\033[7m$1" || out "$1"; pr_off; }
pr_reverse_bold() { [[ "$COLOR" -ne 0 ]] && out "\033[7m\033[1m$1" || out "$1"; pr_off; }
@ -475,7 +475,15 @@ fileout_header() {
else
"$do_json" && printf "[\n" > "$JSONFILE"
fi
"$do_csv" && [[ ! -f "CSVFILE" ]] && echo "\"id\",\"fqdn/ip\",\"port\",\"severity\",\"finding\"" > "$CSVFILE"
if "$do_csv"; then
if [[ -f "$CSVFILE" ]]; then
# add lf, just for overview
echo >> "$CSVFILE"
else
# create file, with headline
echo "\"id\",\"fqdn/ip\",\"port\",\"severity\",\"finding\"" > "$CSVFILE"
fi
fi
else
"$do_json" && printf "[\n" > "$JSONFILE"
"$do_csv" && echo "\"id\",\"fqdn/ip\",\"port\",\"severity\",\"finding\"" > "$CSVFILE"
@ -491,7 +499,7 @@ fileout() { # ID, SEVERITY, FINDING
if "$do_json"; then
"$FIRST_FINDING" || echo -n "," >> $JSONFILE
echo -e " {
echo " {
\"id\" : \"$1\",
\"ip\" : \"$NODE/$NODEIP\",
\"port\" : \"$PORT\",
@ -547,13 +555,17 @@ colon_to_spaces() {
}
strip_lf() {
echo "$1" | tr -d '\n' | tr -d '\r'
tr -d '\n' <<< "$1" | tr -d '\r'
}
strip_spaces() {
echo "${1// /}"
}
trim_trailing_space() {
echo "${1%%*( )}"
}
toupper() {
echo -n "$1" | tr 'a-z' 'A-Z'
}
@ -657,7 +669,7 @@ fi
# determines whether the port has an HTTP service running or not (plain TLS, no STARTTLS)
# arg1 could be the protocol determined as "working". IIS6 needs that
runs_HTTP() {
service_detection() {
local -i ret=0
local -i was_killed
local addcmd=""
@ -693,10 +705,10 @@ runs_HTTP() {
fileout "client_auth" "INFO" "certificate based authentication => skipping all HTTP checks"
else
out " Couldn't determine what's running on port $PORT"
if $ASSUMING_HTTP; then
if "$ASSUME_HTTP"; then
SERVICE=HTTP
out " -- ASSUMING_HTTP set though"
fileout "service" "DEBUG" "Couldn't determine service, --ASSUMING_HTTP set"
out " -- ASSUME_HTTP set though"
fileout "service" "DEBUG" "Couldn't determine service, --ASSUME_HTTP set"
ret=0
else
out ", assuming no HTTP service => skipping all HTTP checks"
@ -1010,11 +1022,16 @@ run_hpkp() {
local -i hpkp_age_sec
local -i hpkp_age_days
local -i hpkp_nr_keys
local hpkp_key hpkp_key_hostcert
local hpkp_spki hpkp_spki_hostcert
local -a backup_spki
local spaces=" "
local key_found=false
local spaces_indented=" "
local certificate_found=false
local i
local hpkp_headers=""
local hpkp_headers
local first_hpkp_header
local spki
local ca_hashes="$TESTSSL_INSTALL_DIR/etc/ca_hashes.txt"
if [[ ! -s $HEADERFILE ]]; then
run_http_header "$1" || return 3
@ -1022,12 +1039,23 @@ run_hpkp() {
pr_bold " Public Key Pinning "
egrep -aiw '^Public-Key-Pins|Public-Key-Pins-Report-Only' $HEADERFILE >$TMPFILE
if [[ $? -eq 0 ]]; then
detect_header "Public-Key-Pins" "HPKP"
hpkp_header=$HEADERVALUE
#FIXME: we should treat report_only seperately
detect_header "Public-Key-Pins-Report-Only" "HPKP_report_only"
hpkp_header+="$HEADERVALUE "
if egrep -aciw '^Public-Key-Pins|Public-Key-Pins-Report-Only' $HEADERFILE | egrep -waq "1" ; then
:
else
hpkp_headers=""
pr_svrty_medium "multiple HPKP headers: "
# https://scotthelme.co.uk is a candidate
#FIXME: should display both Public-Key-Pins+Public-Key-Pins-Report-Only --> egrep -ai -w
for i in $(newline_to_spaces "$(egrep -ai '^Public-Key-Pins' $HEADERFILE | awk -F':' '/Public-Key-Pins/ { print $1 }')"); do
pr_italic $i
hpkp_headers="$hpkp_headers$i "
out " "
done
out "\n$spaces Examining first one: "
first_hpkp_header=$(awk -F':' '/Public-Key-Pins/ { print $1 }' $HEADERFILE | head -1)
pr_italic "$first_hpkp_header, "
fileout "hpkp_multiple" "WARN" "Multiple HPKP headers $hpkp_headers. Using first header: $first_hpkp_header"
fi
# remove leading Public-Key-Pins*, any colons, double quotes and trailing spaces and taking the first -- whatever that is
sed -e 's/Public-Key-Pins://g' -e s'/Public-Key-Pins-Report-Only://' $TMPFILE | \
@ -1037,13 +1065,13 @@ run_hpkp() {
tr ' ' '\n' < $TMPFILE.2 >$TMPFILE
hpkp_nr_keys=$(grep -ac pin-sha $TMPFILE)
out " # of keys: "
if [[ $hpkp_nr_keys -eq 1 ]]; then
pr_svrty_high "1 (NOT ok), "
fileout "hpkp_keys" "NOT ok" "Only one key pinned in HPKP header, this means the site may become unavailable if the key is revoked"
pr_svrty_high "1 key (NOT ok), "
fileout "hpkp_spkis" "HIGH" "Only one key pinned in HPKP header, this means the site may become unavailable if the key is revoked"
else
out "$hpkp_nr_keys, "
fileout "hpkp_keys" "OK" "$hpkp_nr_keys keys pinned in HPKP header, additional keys are available if the current key is revoked"
pr_done_good "$hpkp_nr_keys"
out " keys, "
fileout "hpkp_spkis" "OK" "$hpkp_nr_keys keys pinned in HPKP header, additional keys are available if the current key is revoked"
fi
# print key=value pair with awk, then strip non-numbers, to be improved with proper parsing of key-value with awk
@ -1069,33 +1097,155 @@ run_hpkp() {
fileout "hpkp_preload" "INFO" "HPKP header is NOT marked for browser preloading"
fi
# Get the SPKIs first
spki=$(tr ';' '\n' < $TMPFILE | tr -d ' ' | tr -d '\"' | awk -F'=' '/pin.*=/ { print $2 }')
debugme outln "\n$spki"
# Look at the host certificate first
# get the key fingerprint from the host certificate
if [[ ! -s "$HOSTCERT" ]]; then
get_host_cert || return 1
fi
# get the key fingerprint from the host certificate
hpkp_key_hostcert="$($OPENSSL x509 -in $HOSTCERT -pubkey -noout | grep -v PUBLIC | \
hpkp_spki_hostcert="$($OPENSSL x509 -in $HOSTCERT -pubkey -noout | grep -v PUBLIC | \
$OPENSSL base64 -d | $OPENSSL dgst -sha256 -binary | $OPENSSL base64)"
# compare it with the ones provided in the header
while read hpkp_key; do
if [[ "$hpkp_key_hostcert" == "$hpkp_key" ]] || [[ "$hpkp_key_hostcert" == "$hpkp_key=" ]]; then
out "\n$spaces matching host key: "
pr_done_good "$hpkp_key"
fileout "hpkp_keymatch" "OK" "Key matches a key pinned in the HPKP header"
key_found=true
fi
debugme out "\n $hpkp_key | $hpkp_key_hostcert"
done < <(tr ';' '\n' < $TMPFILE | tr -d ' ' | tr -d '\"' | awk -F'=' '/pin.*=/ { print $2 }')
if ! $key_found ; then
out "\n$spaces"
pr_svrty_high " No matching key for pins found "
out "(CAs pinned? -- not checked for yet)"
fileout "hpkp_keymatch" "DEBUG" "The TLS key does not match any key pinned in the HPKP header. If you pinned a CA key you can ignore this"
hpkp_ca="$($OPENSSL x509 -in $HOSTCERT -issuer -noout|sed 's/^.*CN=//' | sed 's/\/.*$//')"
# Get keys/hashes from intermediate certificates
$OPENSSL s_client -showcerts $STARTTLS $BUGS $PROXY -showcerts -connect $NODEIP:$PORT ${sni[i]} </dev/null >$TMPFILE 2>$ERRFILE
# Place the server's certificate in $HOSTCERT and any intermediate
# certificates that were provided in $TEMPDIR/intermediatecerts.pem
# http://backreference.org/2010/05/09/ocsp-verification-with-openssl/
awk -v n=-1 "/Certificate chain/ {start=1}
/-----BEGIN CERTIFICATE-----/{ if (start) {inc=1; n++} }
inc { print > (\"$TEMPDIR/level\" n \".crt\") }
/---END CERTIFICATE-----/{ inc=0 }" $TMPFILE
nrsaved=$(count_words "$(echo $TEMPDIR/level?.crt 2>/dev/null)")
rm $TEMPDIR/level0.crt 2>/dev/null
printf ""> "$TEMPDIR/intermediate.hashes"
if [[ nrsaved -ge 2 ]]; then
for cert_fname in $TEMPDIR/level?.crt; do
hpkp_spki_ca="$($OPENSSL x509 -in "$cert_fname" -pubkey -noout | grep -v PUBLIC | $OPENSSL base64 -d |
$OPENSSL dgst -sha256 -binary | $OPENSSL enc -base64)"
hpkp_name="$(get_cn_from_cert $cert_fname)"
hpkp_ca="$($OPENSSL x509 -in $cert_fname -issuer -noout|sed 's/^.*CN=//' | sed 's/\/.*$//')"
[[ -n $hpkp_name ]] || hpkp_name=$($OPENSSL x509 -in "$cert_fname" -subject -noout | sed 's/^subject= //')
echo "$hpkp_spki_ca $hpkp_name" >> "$TEMPDIR/intermediate.hashes"
done
fi
# This is where the matching magic starts, first host certificate, intermediate, then root out of the stores
spki_match=false
has_backup_spki=false
i=0
for hpkp_spki in $spki; do
certificate_found=false
# compare collected SPKIs against the host certificate
if [[ "$hpkp_spki_hostcert" == "$hpkp_spki" ]] || [[ "$hpkp_spki_hostcert" == "$hpkp_spki=" ]]; then
certificate_found=true # We have a match
spki_match=true
out "\n$spaces_indented Host cert: "
pr_done_good "$hpkp_spki"
fileout "hpkp_$hpkp_spki" "OK" "SPKI $hpkp_spki matches the host certificate"
fi
debugme out "\n $hpkp_spki | $hpkp_spki_hostcert"
# Check for intermediate match
if ! "$certificate_found"; then
hpkp_matches=$(grep "$hpkp_spki" $TEMPDIR/intermediate.hashes 2>/dev/null)
if [[ -n $hpkp_matches ]]; then # hpkp_matches + hpkp_spki + '='
# We have a match
certificate_found=true
spki_match=true
out "\n$spaces_indented Sub CA: "
pr_done_good "$hpkp_spki"
ca_cn="$(sed "s/^[a-zA-Z0-9\+\/]*=* *//" <<< $"$hpkp_matches" )"
pr_italic " $ca_cn"
fileout "hpkp_$hpkp_spki" "OK" "SPKI $hpkp_spki matches Intermediate CA \"$ca_cn\" pinned in the HPKP header"
fi
fi
# we compare now against a precompiled list of SPKIs against the ROOT CAs we have in $ca_hashes
if ! "$certificate_found"; then
hpkp_matches=$(grep -h "$hpkp_spki" $ca_hashes | sort -u)
if [[ -n $hpkp_matches ]]; then
certificate_found=true # root CA found
spki_match=true
if [[ $(count_lines "$hpkp_matches") -eq 1 ]]; then
# replace by awk
match_ca=$(sed "s/[a-zA-Z0-9\+\/]*=* *//" <<< "$hpkp_matches")
else
match_ca=""
fi
ca_cn="$(sed "s/^[a-zA-Z0-9\+\/]*=* *//" <<< $"$hpkp_matches" )"
if [[ "$match_ca" == "$hpkp_ca" ]]; then # part of the chain
out "\n$spaces_indented Root CA: "
pr_done_good "$hpkp_spki"
pr_italic " $ca_cn"
fileout "hpkp_$hpkp_spki" "INFO" "SPKI $hpkp_spki matches Root CA \"$ca_cn\" pinned in the HPKP header. (Root CA part of the chain)"
else # not part of chain
match_ca=""
has_backup_spki=true # Root CA outside the chain --> we save it for unmatched
fileout "hpkp_$hpkp_spki" "INFO" "SPKI $hpkp_spki matches Root CA \"$ca_cn\" pinned in the HPKP header. (Root backup SPKI)"
backup_spki[i]="$(strip_lf "$hpkp_spki")" # save it for later
backup_spki_str[i]="$ca_cn" # also the name=CN of the root CA
i=$((i + 1))
fi
fi
fi
# still no success --> it's probably a backup SPKI
if ! "$certificate_found"; then
# Most likely a backup SPKI, unfortunately we can't tell for what it is: host, intermediates
has_backup_spki=true
backup_spki[i]="$(strip_lf "$hpkp_spki")" # save it for later
backup_spki_str[i]="" # no root ca
i=$((i + 1))
fileout "hpkp_$hpkp_spki" "INFO" "SPKI $hpkp_spki doesn't match anything. This is ok for a backup for any certificate"
# CSV/JSON output here for the sake of simplicity, rest we do en bloc below
fi
done
# now print every backup spki out we saved before
out "\n$spaces_indented Backups: "
# for i=0 manually do the same as below as there's other indentation here
if [[ -n "${backup_spki_str[0]}" ]]; then
pr_done_good "${backup_spki[0]}"
#out " Root CA: "
pr_italicln " ${backup_spki_str[0]}"
else
outln "${backup_spki[0]}"
fi
# now for i=1
for ((i=1; i < ${#backup_spki[@]} ;i++ )); do
if [[ -n "${backup_spki_str[i]}" ]]; then
# it's a Root CA outside the chain
pr_done_good "$spaces_indented ${backup_spki[i]}"
#out " Root CA: "
pr_italicln " ${backup_spki_str[i]}"
else
outln "$spaces_indented ${backup_spki[i]}"
fi
done
# If all else fails...
if ! "$spki_match"; then
"$has_backup_spki" && out "$spaces" # we had a few lines with backup SPKIs already
pr_svrty_highln " No matching key for SPKI found "
fileout "hpkp_spkimatch" "HIGH" "None of the SPKI match your host certificate, intermediate CA or known root CAs. You may have bricked this site"
fi
if ! "$has_backup_spki"; then
pr_svrty_highln " No backup keys found. Loss/compromise of the currently pinned key(s) will lead to bricked site. "
fileout "hpkp_backup" "HIGH" "No backup keys found. Loss/compromise of the currently pinned key(s) will lead to bricked site."
fi
else
out "--"
outln "--"
fileout "hpkp" "INFO" "No support for HTTP Public Key Pinning"
fi
outln
tmpfile_handle $FUNCNAME.txt
return $?
@ -3019,7 +3169,6 @@ run_protocols() {
fi
fi ;;
esac
pr_off
debugme outln
else
run_prototest_openssl "-ssl2"
@ -5078,7 +5227,7 @@ run_pfs() {
spdy_pre(){
if [[ -n "$STARTTLS" ]]; then
if [[ -n "$STARTTLS" ]] || [[ "$SERVICE" != HTTP ]]; then
[[ -n "$1" ]] && out "$1"
out "(SPDY is an HTTP protocol and thus not tested here)"
fileout "spdy_npn" "INFO" "SPDY/NPN : (SPY is an HTTP protocol and thus not tested here)"
@ -5099,7 +5248,7 @@ spdy_pre(){
}
http2_pre(){
if [[ -n "$STARTTLS" ]]; then
if [[ -n "$STARTTLS" ]] || [[ "$SERVICE" != HTTP ]]; then
[[ -n "$1" ]] && out "$1"
outln "(HTTP/2 is a HTTP protocol and thus not tested here)"
fileout "https_alpn" "INFO" "HTTP2/ALPN : HTTP/2 is and HTTP protocol and thus not tested"
@ -7292,7 +7441,6 @@ check4openssl_oldfarts() {
outln
}
# FreeBSD needs to have /dev/fd mounted. This is a friendly hint, see #258
check_bsd_mount() {
if [[ "$(uname)" == FreeBSD ]]; then
@ -7306,7 +7454,6 @@ check_bsd_mount() {
fi
}
help() {
cat << EOF
@ -7366,7 +7513,7 @@ partly mandatory parameters:
tuning options (can also be preset via environment variables):
--bugs enables the "-bugs" option of s_client, needed e.g. for some buggy F5s
--assuming-http if protocol check fails it assumes HTTP protocol and enforces HTTP checks
--assume-http if protocol check fails it assumes HTTP protocol and enforces HTTP checks
--ssl-native fallback to checks with OpenSSL where sockets are normally used
--openssl <PATH> use this openssl binary (default: look in \$PATH, \$RUN_DIR of $PROG_NAME)
--proxy <host>:<port> connect via the specified HTTP proxy
@ -7468,7 +7615,7 @@ HAS_SED_E: $HAS_SED_E
SHOW_EACH_C: $SHOW_EACH_C
SSL_NATIVE: $SSL_NATIVE
ASSUMING_HTTP $ASSUMING_HTTP
ASSUME_HTTP $ASSUME_HTTP
SNEAKY: $SNEAKY
DEBUG: $DEBUG
@ -8074,10 +8221,10 @@ determine_service() {
ua="$UA_SNEAKY" || \
ua="$UA_STD"
GET_REQ11="GET $URL_PATH HTTP/1.1\r\nHost: $NODE\r\nUser-Agent: $ua\r\nConnection: Close\r\nAccept: text/*\r\n\r\n"
HEAD_REQ11="HEAD $URL_PATH HTTP/1.1\r\nHost: $NODE\r\nUser-Agent: $ua\r\nAccept: text/*\r\n\r\n"
GET_REQ10="GET $URL_PATH HTTP/1.0\r\nUser-Agent: $ua\r\nConnection: Close\r\nAccept: text/*\r\n\r\n"
HEAD_REQ10="HEAD $URL_PATH HTTP/1.0\r\nUser-Agent: $ua\r\nAccept: text/*\r\n\r\n"
runs_HTTP $OPTIMAL_PROTO
#HEAD_REQ11="HEAD $URL_PATH HTTP/1.1\r\nHost: $NODE\r\nUser-Agent: $ua\r\nAccept: text/*\r\n\r\n"
#GET_REQ10="GET $URL_PATH HTTP/1.0\r\nUser-Agent: $ua\r\nConnection: Close\r\nAccept: text/*\r\n\r\n"
#HEAD_REQ10="HEAD $URL_PATH HTTP/1.0\r\nUser-Agent: $ua\r\nAccept: text/*\r\n\r\n"
service_detection $OPTIMAL_PROTO
else
# STARTTLS
protocol=${1%s} # strip trailing 's' in ftp(s), smtp(s), pop3(s), etc
@ -8547,7 +8694,7 @@ parse_cmd_line() {
WIDE=true
;;
--assuming[_-]http|--assume[-_]http)
ASSUMING_HTTP=true
ASSUME_HTTP=true
;;
--sneaky)
SNEAKY=true
@ -8844,4 +8991,4 @@ fi
exit $?
# $Id: testssl.sh,v 1.557 2016/10/10 21:27:33 dirkw Exp $
# $Id: testssl.sh,v 1.559 2016/10/15 20:55:22 dirkw Exp $