From cfecc3c8c4aa4fd15cc0240a23c48106d6f06ee4 Mon Sep 17 00:00:00 2001 From: David Cooper Date: Thu, 14 Dec 2017 10:18:24 -0500 Subject: [PATCH 1/8] Derive handshake traffic key for TLSv1.3 This is the first in a series of PRs to add support for processing the encrypted portions of the server's response in a TLSv1.3 handshake. This PR adds the code to derive the handshake traffic key needed to decrypt the response (the next PR will add the code to perform the symmetric-key decryption of the encrypted portions of the response). Since this PR does not make use of the traffic key that it derives, it doesn't yet add any new functionality. Note that testssl.sh will not always be able to derive the session keys. If the version of OpenSSL that is bundled with testssl.sh is used and the server chooses to use an X25519 ephemeral key, OpenSSL will be unable to perform the shared secret in derive-handshake-traffic-secret(). (OpenSSL 1.1.0 supports X25519.) Since X25519 use a different encoding than ECDH keys, the lack of X25519 support will be discovered in parse_tls_serverhello() when $OPENSSL pkey is unable to convert the key from DER to PEM. So, in debugging mode, parse_tls_serverhello() now displays a warning if it receives a key share that $OPENSSL pkey cannot handle. --- testssl.sh | 280 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 279 insertions(+), 1 deletion(-) diff --git a/testssl.sh b/testssl.sh index 3ce514d..9f8408c 100755 --- a/testssl.sh +++ b/testssl.sh @@ -8491,6 +8491,257 @@ parse_sslv2_serverhello() { return $ret } +# arg1: hash function +# arg2: key +# arg3: text +hmac() { + local hash_fn="$1" + local key="$2" text="$3" output + local -i ret + + output="$(asciihex_to_binary_file "$text" "/dev/stdout" | $OPENSSL dgst "$hash_fn" -mac HMAC -macopt hexkey:"$key" 2>/dev/null)" + ret=$? + tm_out "$(awk '/=/ { print $2 }' <<< "$output")" + return $ret +} + +# arg1: hash function +# arg2: pseudorandom key (PRK) +# arg2: info +# arg3: length of output keying material in octets +# See RFC 5869, Section 2.3 +hkdf-expand() { + local hash_fn="$1" + local prk="$2" info="$3" output="" + local -i out_len="$4" + local -i i n mod_check hash_len ret + local counter + local ti tim1 # T(i) and T(i-1) + + case "$hash_fn" in + "-sha256") hash_len=32 ;; + "-sha384") hash_len=48 ;; + *) return 7 + esac + + n=$out_len/$hash_len + mod_check=$out_len%$hash_len + [[ $mod_check -ne 0 ]] && n+=1 + + tim1="" + for (( i=1; i <= n; i++ )); do + counter="$(printf "%02X\n" $i)" + ti="$(hmac "$hash_fn" "$prk" "$tim1$info$counter")" + [[ $? -ne 0 ]] && return 7 + output+="$ti" + tim1="$ti" + done + out_len=2*$out_len + tm_out "${output:0:out_len}" + return 0 +} + +# arg1: hash function +# arg2: secret +# arg3: label +# arg4: context +# arg5: length +# See draft-ietf-tls-tls13, Section 7.1 +hkdf-expand-label() { + local hash_fn="$1" + local secret="$2" label="$3" + local context="$4" + local -i length="$5" + local hkdflabel hkdflabel_label hkdflabel_context + local hkdflabel_length + local -i len + + hkdflabel_length="$(printf "%04X\n" $length)" + if [[ "${TLS_SERVER_HELLO:8:2}" == "7F" ]] && [[ 0x${TLS_SERVER_HELLO:10:2} -lt 0x14 ]]; then + # "544c5320312e332c20" = "TLS 1.3, " + hkdflabel_label="544c5320312e332c20$label" + else + # "746c73313320" = "tls13 " + hkdflabel_label="746c73313320$label" + fi + len=${#hkdflabel_label}/2 + hkdflabel_label="$(printf "%02X\n" $len)$hkdflabel_label" + len=${#context}/2 + hkdflabel_context="$(printf "%02X\n" $len)$context" + hkdflabel="$hkdflabel_length$hkdflabel_label$hkdflabel_context" + + hkdf-expand "$hash_fn" "$secret" "$hkdflabel" "$length" + return $? +} + +# arg1: hash function +# arg2: secret +# arg3: label +# arg4: ASCII-HEX of messages +# See draft-ietf-tls-tls13, Section 7.1 +derive-secret() { + local hash_fn="$1" + local secret="$2" label="$3" messages="$4" + local hash_messages + local -i hash_len retcode + + case "$hash_fn" in + "-sha256") hash_len=32 ;; + "-sha384") hash_len=48 ;; + *) return 7 + esac + + hash_messages="$(asciihex_to_binary_file "$messages" "/dev/stdout" | $OPENSSL dgst "$hash_fn" 2>/dev/null | awk '/=/ { print $2 }')" + hkdf-expand-label "$hash_fn" "$secret" "$label" "$hash_messages" "$hash_len" + return $? +} + +# arg1: hash function +# arg2: private key file +# arg3: file containing server's ephemeral public key +# arg4: ASCII-HEX of messages (ClientHello...ServerHello) +# See key derivation schedule diagram in Section 7.1 of draft-ietf-tls-tls13 +derive-handshake-traffic-secret() { + local hash_fn="$1" + local priv_file="$2" pub_file="$3" + local messages="$4" + local -i i ret + local early_secret derived_secret shared_secret handshake_secret + + "$HAS_PKUTIL" || return 1 + + # early_secret="$(hmac "$hash_fn" "000...000" "000...000")" + case "$hash_fn" in + "-sha256") early_secret="33ad0a1c607ec03b09e6cd9893680ce210adf300aa1f2660e1b22e10f170f92a" + if [[ "${TLS_SERVER_HELLO:8:2}" == "7F" ]] && [[ 0x${TLS_SERVER_HELLO:10:2} -lt 0x14 ]]; then + # "6465726976656420736563726574" = "derived secret" + # derived_secret="$(derive-secret "$hash_fn" "$early_secret" "6465726976656420736563726574" "")" + derived_secret="c1c0c36bf8fb1d1afa949fbd360e71af69a6244a4c2eaef5bbbb6442a7277d2c" + else + # "64657269766564" = "derived" + # derived_secret="$(derive-secret "$hash_fn" "$early_secret" "64657269766564" "")" + derived_secret="6f2615a108c702c5678f54fc9dbab69716c076189c48250cebeac3576c3611ba" + fi + ;; + "-sha384") early_secret="7ee8206f5570023e6dc7519eb1073bc4e791ad37b5c382aa10ba18e2357e716971f9362f2c2fe2a76bfd78dfec4ea9b5" + if [[ "${TLS_SERVER_HELLO:8:2}" == "7F" ]] && [[ 0x${TLS_SERVER_HELLO:10:2} -lt 0x14 ]]; then + # "6465726976656420736563726574" = "derived secret" + # derived_secret="$(derive-secret "$hash_fn" "$early_secret" "6465726976656420736563726574" "")" + derived_secret="54c80fa05ee9e0532ce3db8ddeca37a0365683bcd3b27bdc88d2b9fdc115ca4ebc8edc1f0b72a6a0861e803fc34761ef" + else + # "64657269766564" = "derived" + # derived_secret="$(derive-secret "$hash_fn" "$early_secret" "64657269766564" "")" + derived_secret="1591dac5cbbf0330a4a84de9c753330e92d01f0a88214b4464972fd668049e93e52f2b16fad922fdc0584478428f282b" + fi + ;; + *) return 7 + esac + + shared_secret="$($OPENSSL pkeyutl -derive -inkey "$priv_file" -peerkey "$pub_file" 2>/dev/null | hexdump -v -e '16/1 "%02X"')" + + # For draft 18 use $early_secret rather than $derived_secret. + if [[ "${TLS_SERVER_HELLO:8:4}" == "7F12" ]]; then + handshake_secret="$(hmac "$hash_fn" "$early_secret" "${shared_secret%%[!0-9A-F]*}")" + else + handshake_secret="$(hmac "$hash_fn" "$derived_secret" "${shared_secret%%[!0-9A-F]*}")" + fi + [[ $? -ne 0 ]] && return 7 + + if [[ "${TLS_SERVER_HELLO:8:2}" == "7F" ]] && [[ 0x${TLS_SERVER_HELLO:10:2} -lt 0x14 ]]; then + # "7365727665722068616e647368616b65207472616666696320736563726574" = "server handshake traffic secret" + derived_secret="$(derive-secret "$hash_fn" "$handshake_secret" "7365727665722068616e647368616b65207472616666696320736563726574" "$messages")" + else + # "732068732074726166666963" = "s hs traffic" + derived_secret="$(derive-secret "$hash_fn" "$handshake_secret" "732068732074726166666963" "$messages")" + fi + [[ $? -ne 0 ]] && return 7 + tm_out "$derived_secret" + return 0 +} + +# arg1: hash function +# arg2: secret (created by derive-handshake-traffic-secret) +# arg3: purpose ("key" or "iv") +# arg4: length of the key +# See draft-ietf-tls-tls13, Section 7.3 +derive-traffic-key() { + local hash_fn="$1" + local secret="$2" purpose="$3" + local -i key_length="$4" + local key + + key="$(hkdf-expand-label "$hash_fn" "$secret" "$purpose" "" "$key_length")" + [[ $? -ne 0 ]] && return 7 + tm_out "$key" + return 0 +} + +#arg1: TLS cipher +#arg2: file containing cipher name, public key, and private key +#arg3: First ClientHello, if response was a HelloRetryRequest +#arg4: HelloRetryRequest, if one was sent +#arg5: Final (or only) ClientHello +#arg6: ServerHello +derive-handshake-traffic-keys() { + local cipher="$1" + local tmpfile="$2" + local clienthello1="$3" hrr="$4" clienthello2="$5" serverhello="$6" + local hash_clienthello1 + local -i key_len + local -i retcode + local hash_fn + local pub_file priv_file tmpfile + + if [[ "$cipher" == *SHA256 ]]; then + hash_fn="-sha256" + elif [[ "$cipher" == *SHA384 ]]; then + hash_fn="-sha384" + else + return 1 + fi + if [[ "$cipher" == *AES_128* ]]; then + key_len=16 + elif ( [[ "$cipher" == *AES_256* ]] || [[ "$cipher" == *CHACHA20_POLY1305* ]] ); then + key_len=32 + else + return 1 + fi + pub_file="$(mktemp "$TEMPDIR/pubkey.XXXXXX")" || return 7 + awk '/-----BEGIN PUBLIC KEY/,/-----END PUBLIC KEY/ { print $0 }' \ + "$tmpfile" > "$pub_file" + [[ ! -s "$pub_file" ]] && return 1 + + priv_file="$(mktemp "$TEMPDIR/privkey.XXXXXX")" || return 7 + if grep -q "\-\-\-\-\-BEGIN EC PARAMETERS" "$tmpfile"; then + awk '/-----BEGIN EC PARAMETERS/,/-----END EC PRIVATE KEY/ { print $0 }' \ + "$tmpfile" > "$priv_file" + else + awk '/-----BEGIN PRIVATE KEY/,/-----END PRIVATE KEY/ { print $0 }' \ + "$tmpfile" > "$priv_file" + fi + [[ ! -s "$priv_file" ]] && return 1 + + if [[ -n "$hrr" ]] && [[ "${serverhello:8:4}" == "7F12" ]]; then + derived_secret="$(derive-handshake-traffic-secret "$hash_fn" "$priv_file" "$pub_file" "$clienthello1$hrr$clienthello2$serverhello")" + elif [[ -n "$hrr" ]]; then + hash_clienthello1="$(asciihex_to_binary_file "$clienthello1" "/dev/stdout" | $OPENSSL dgst "$hash_fn" 2>/dev/null | awk '/=/ { print $2 }')" + derived_secret="$(derive-handshake-traffic-secret "$hash_fn" "$priv_file" "$pub_file" "FE0000$(printf "%02x" $((${#hash_clienthello1}/2)))$hash_clienthello1$hrr$clienthello2$serverhello")" + else + derived_secret="$(derive-handshake-traffic-secret "$hash_fn" "$priv_file" "$pub_file" "$clienthello2$serverhello")" + fi + retcode=$? + rm $pub_file $priv_file + [[ $retcode -ne 0 ]] && return 1 + # "6b6579" = "key" + server_write_key="$(derive-traffic-key "$hash_fn" "$derived_secret" "6b6579" "$key_len")" + [[ $? -ne 0 ]] && return 1 + # "6976" = "iv" + server_write_iv="$(derive-traffic-key "$hash_fn" "$derived_secret" "6976" "12")" + [[ $? -ne 0 ]] && return 1 + tm_out "$server_write_key $server_write_iv" + return 0 +} + # Return: # 0 if arg1 contains the entire server response. # 1 if arg1 does not contain the entire server response. @@ -9142,6 +9393,13 @@ parse_tls_serverhello() { fi if [[ -n "$key_bitstring" ]]; then key_bitstring="$(asciihex_to_binary_file "$key_bitstring" "/dev/stdout" | $OPENSSL pkey -pubin -inform DER 2>$ERRFILE)" + if [[ -z "$key_bitstring" ]] && [[ $DEBUG -ge 2 ]]; then + if [[ -n "$named_curve_str" ]]; then + prln_warning "Your $OPENSSL doesn't support $named_curve_str" + else + prln_warning "Your $OPENSSL doesn't support named curve $named_curve" + fi + fi fi fi ;; @@ -9585,6 +9843,8 @@ parse_tls_serverhello() { fi fi tmpfile_handle $FUNCNAME.txt + + TLS_SERVER_HELLO="02$(printf "%06x" $(($tls_serverhello_ascii_len/2)))${tls_serverhello_ascii}" return 0 } @@ -10090,7 +10350,7 @@ socksend_tls_clienthello() { printf -- "$data" >&5 2>/dev/null & sleep $USLEEP_SND - if [[ "$tls_low_byte" -gt 0x03 ]]; then + if [[ "$tls_low_byte" -gt 0x03 ]]; then TLS_CLIENT_HELLO="$(tolower "$NW_STR")" TLS_CLIENT_HELLO="${TLS_CLIENT_HELLO//\\x0\\/\\x00\\}" TLS_CLIENT_HELLO="${TLS_CLIENT_HELLO//\\x1\\/\\x01\\}" @@ -10338,9 +10598,11 @@ tls_sockets() { local cipher_list_2send local sock_reply_file2 sock_reply_file3 local tls_hello_ascii next_packet + local clienthello1 hrr="" local process_full="$3" offer_compression=false skip=false local close_connection=true local -i hello_done=0 + local cipher="" key_and_iv="" [[ "$5" == "true" ]] && offer_compression=true [[ "$6" == "false" ]] && close_connection=false @@ -10363,6 +10625,7 @@ tls_sockets() { # if sending didn't succeed we don't bother if [[ $ret -eq 0 ]]; then + clienthello1="$TLS_CLIENT_HELLO" sockread_serverhello 32768 "$TLS_DIFFTIME_SET" && TLS_NOW=$(LC_ALL=C date "+%s") @@ -10373,6 +10636,7 @@ tls_sockets() { resend_if_hello_retry_request "$tls_hello_ascii" "$cipher_list_2send" "$4" "$process_full" ret=$? if [[ $ret -eq 2 ]]; then + hrr="${tls_hello_ascii:10}" tls_hello_ascii=$(hexdump -v -e '16/1 "%02X"' "$SOCK_REPLY_FILE") tls_hello_ascii="${tls_hello_ascii%%[!0-9A-F]*}" elif [[ $ret -eq 1 ]] || [[ $ret -eq 6 ]]; then @@ -10429,6 +10693,20 @@ tls_sockets() { check_tls_serverhellodone "$tls_hello_ascii" "$process_full" hello_done=$? if [[ "$hello_done" -eq 3 ]]; then + debugme echo "reading server hello..." + parse_tls_serverhello "$tls_hello_ascii" "ephemeralkey" + ret=$? + if [[ "$ret" -eq 0 ]] || [[ "$ret" -eq 2 ]]; then + cipher=$(get_cipher "$TEMPDIR/$NODEIP.parse_tls_serverhello.txt") + if [[ -n "$hrr" ]]; then + key_and_iv="$(derive-handshake-traffic-keys "$cipher" "$TEMPDIR/$NODEIP.parse_tls_serverhello.txt" "$clienthello1" "$hrr" "$TLS_CLIENT_HELLO" "$TLS_SERVER_HELLO")" + else + key_and_iv="$(derive-handshake-traffic-keys "$cipher" "$TEMPDIR/$NODEIP.parse_tls_serverhello.txt" "" "" "$TLS_CLIENT_HELLO" "$TLS_SERVER_HELLO")" + fi + [[ $? -ne 0 ]] && hello_done=2 + else + hello_done=2 + fi # The following three lines are temporary until the code # to decrypt TLSv1.3 responses has been added, at which point # parse_tls_serverhello() will be called with process_full="all" From e8be1f441b0a76d7c3440374441d9e767e193709 Mon Sep 17 00:00:00 2001 From: David Cooper Date: Fri, 15 Dec 2017 16:40:47 -0500 Subject: [PATCH 2/8] Decrypt server's TLSv1.3 response This PR adds code to decrypt the encrypted portion of the server's response for TLSv1.3 and to then process any certificates and encrypted extensions. This code supports all 5 TLSv1.3 cipher suites, and so any response can be decrypted as long as the session key can be derived (which requires OpenSSL to support the ephemeral key that was used - see #938). For the symmetric decryption, the sym-decrypt() function uses OpenSSL when possible and internal Bash functions when needed. For AES-GCM and AES-CCM ciphers sym-decrypt() normally uses internal Bash functions, which rely on using "$OPENSSL enc" in AES-ECB mode to generate the key stream and then Bash functionality to XOR the key stream with the ciphertext. With some version of OpenSSL the AES-GCM ciphers are decrypted using "$OPENSSL enc" in AES-GCM mode directly. On my system, however, both methods seem to work about equally fast. For ChaCha20 ciphers, "$OPENSSL enc -chacha20" is used, if supported (OpenSSL 1.1.x only). and Bash internal functions (without any OpenSSL support) are used otherwise. In this case, if the Bash internal functions need to be used, decryption is very, very, very slow. Fortunately, in a typical run of testssl.sh there won't be many cases in which the connection will be TLSv1.3 with ChaCha20 and the entire response needs to be processed (requiring decryption). In most cases, even if the connection is TLSv1.3 with ChaCha20, will at most need the ephemeral key, which is available in plain text. --- testssl.sh | 497 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 478 insertions(+), 19 deletions(-) diff --git a/testssl.sh b/testssl.sh index 9f8408c..00e248a 100755 --- a/testssl.sh +++ b/testssl.sh @@ -286,6 +286,9 @@ HAS_PROXY=false HAS_XMPP=false HAS_POSTGRES=false HAS_MYSQL=false +HAS_CHACHA20=false +HAS_AES128_GCM=false +HAS_AES256_GCM=false PORT=443 # unless otherwise auto-determined, see below NODE="" NODEIP="" @@ -8742,27 +8745,375 @@ derive-handshake-traffic-keys() { return 0 } +generate-ccm-gcm-keystream() { + local icb="$1" icb_msb icb_lsb1 + local -i i icb_lsb n="$2" + + icb_msb="${icb:0:24}" + icb_lsb=0x${icb:24:8} + + for (( i=0; i < n; i=i+1 )); do + icb_lsb1="$(printf "%08X" $icb_lsb)" + printf "\x${icb_msb:0:2}\x${icb_msb:2:2}\x${icb_msb:4:2}\x${icb_msb:6:2}\x${icb_msb:8:2}\x${icb_msb:10:2}\x${icb_msb:12:2}\x${icb_msb:14:2}\x${icb_msb:16:2}\x${icb_msb:18:2}\x${icb_msb:20:2}\x${icb_msb:22:2}\x${icb_lsb1:0:2}\x${icb_lsb1:2:2}\x${icb_lsb1:4:2}\x${icb_lsb1:6:2}" + icb_lsb+=1 + done + return 0 +} + +# arg1: an OpenSSL ecb cipher (e.g., -aes-128-ecb) +# arg2: key +# arg3: initial counter value (must be 128 bits) +# arg4: ciphertext +# See Sections 6.5 and 7.2 of SP 800-38D and Section 6.2 and Appendix A of SP 800-38C +ccm-gcm-decrypt() { + local cipher="$1" + local key="$2" + local icb="$3" + local ciphertext="$4" + local -i i i1 i2 i3 i4 + local -i ciphertext_len n mod_check + local y plaintext="" + + [[ ${#icb} -ne 32 ]] && return 7 + + ciphertext_len=${#ciphertext} + n=$ciphertext_len/32 + mod_check=$ciphertext_len%32 + [[ $mod_check -ne 0 ]] && n+=1 + y="$(generate-ccm-gcm-keystream "$icb" "$n" | $OPENSSL enc "$cipher" -K "$key" -nopad 2>/dev/null | hexdump -v -e '16/1 "%02X"')" + + # XOR the ciphertext with the keystream ($y). For efficiency, work in blocks of 16 bytes at a time (but with each XOR operation working on + # 32 bits. + [[ $mod_check -ne 0 ]] && n=$n-1 + for (( i=0; i < n; i++ )); do + i1=32*$i; i2=$i1+8; i3=$i1+16; i4=$i1+24 + plaintext+="$(printf "%08X%08X%08X%08X" "$((0x${ciphertext:i1:8} ^ 0x${y:i1:8}))" "$((0x${ciphertext:i2:8} ^ 0x${y:i2:8}))" "$((0x${ciphertext:i3:8} ^ 0x${y:i3:8}))" "$((0x${ciphertext:i4:8} ^ 0x${y:i4:8}))")" + done + # If the length of the ciphertext is not an even multiple of 16 bytes, then handle the final incomplete block. + if [[ $mod_check -ne 0 ]]; then + i1=32*$n + for (( i=0; i < mod_check; i=i+2 )); do + plaintext+="$(printf "%02X" "$((0x${ciphertext:i1:2} ^ 0x${y:i1:2}))")" + i1+=2 + done + fi + tm_out "$plaintext" + return 0 +} + +# See RFC 7539, Section 2.1 +chacha20_Qround() { + local -i a="0x$1" + local -i b="0x$2" + local -i c="0x$3" + local -i d="0x$4" + local -i x y + + a=$(((a+b) & 0xffffffff)) + d=$((d^a)) + # rotate d left 16 bits + x=$((d & 0xffff0000)) + x=$((x >> 16)) + y=$((d & 0x0000ffff)) + y=$((y << 16)) + d=$((x | y)) + + c=$(((c+d) & 0xffffffff)) + b=$((b^c)) + # rotate b left 12 bits + x=$((b & 0xfff00000)) + x=$((x >> 20)) + y=$((b & 0x000fffff)) + y=$((y << 12)) + b=$((x | y)) + + a=$(((a+b) & 0xffffffff)) + d=$((d^a)) + # rotate d left 8 bits + x=$((d & 0xff000000)) + x=$((x >> 24)) + y=$((d & 0x00ffffff)) + y=$((y << 8)) + d=$((x | y)) + + c=$(((c+d) & 0xffffffff)) + b=$((b^c)) + # rotate b left 7 bits + x=$((b & 0xfe000000)) + x=$((x >> 25)) + y=$((b & 0x01ffffff)) + y=$((y << 7)) + b=$((x | y)) + + tm_out "$(printf "%x" $a) $(printf "%x" $b) $(printf "%x" $c) $(printf "%x" $d)" + return 0 +} + +# See RFC 7539, Section 2.3.1 +chacha20_inner_block() { + local s0="$1" s1="$2" s2="$3" s3="$4" + local s4="$5" s5="$6" s6="$7" s7="$8" + local s8="$9" s9="${10}" s10="${11}" s11="${12}" + local s12="${13}" s13="${14}" s14="${15}" s15="${16}" + local res + + res="$(chacha20_Qround "$s0" "$s4" "$s8" "$s12")" + read s0 s4 s8 s12 <<< "$res" + res="$(chacha20_Qround "$s1" "$s5" "$s9" "$s13")" + read s1 s5 s9 s13 <<< "$res" + res="$(chacha20_Qround "$s2" "$s6" "$s10" "$s14")" + read s2 s6 s10 s14 <<< "$res" + res="$(chacha20_Qround "$s3" "$s7" "$s11" "$s15")" + read s3 s7 s11 s15 <<< "$res" + res="$(chacha20_Qround "$s0" "$s5" "$s10" "$s15")" + read s0 s5 s10 s15 <<< "$res" + res="$(chacha20_Qround "$s1" "$s6" "$s11" "$s12")" + read s1 s6 s11 s12 <<< "$res" + res="$(chacha20_Qround "$s2" "$s7" "$s8" "$s13")" + read s2 s7 s8 s13 <<< "$res" + res="$(chacha20_Qround "$s3" "$s4" "$s9" "$s14")" + read s3 s4 s9 s14 <<< "$res" + + tm_out "$s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s8 $s9 $s10 $s11 $s12 $s13 $s14 $s15" + return 0 +} + +# See RFC 7539, Sections 2.3 and 2.3.1 +chacha20_block() { + local key="$1" + local counter="$2" + local nonce="$3" + local s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 + local ws0 ws1 ws2 ws3 ws4 ws5 ws6 ws7 ws8 ws9 ws10 ws11 ws12 ws13 ws14 ws15 + local working_state + local -i i + + # create the state variable + s0="61707865"; s1="3320646e"; s2="79622d32"; s3="6b206574" + s4="${key:6:2}${key:4:2}${key:2:2}${key:0:2}" + s5="${key:14:2}${key:12:2}${key:10:2}${key:8:2}" + s6="${key:22:2}${key:20:2}${key:18:2}${key:16:2}" + s7="${key:30:2}${key:28:2}${key:26:2}${key:24:2}" + s8="${key:38:2}${key:36:2}${key:34:2}${key:32:2}" + s9="${key:46:2}${key:44:2}${key:42:2}${key:40:2}" + s10="${key:54:2}${key:52:2}${key:50:2}${key:48:2}" + s11="${key:62:2}${key:60:2}${key:58:2}${key:56:2}" + s12="$counter" + s13="${nonce:6:2}${nonce:4:2}${nonce:2:2}${nonce:0:2}" + s14="${nonce:14:2}${nonce:12:2}${nonce:10:2}${nonce:8:2}" + s15="${nonce:22:2}${nonce:20:2}${nonce:18:2}${nonce:16:2}" + + # Initialize working_state to state + working_state="$s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s8 $s9 $s10 $s11 $s12 $s13 $s14 $s15" + + # compute the 20 rounds (10 calls to inner block function, each of which + # performs 8 quarter rounds). + for (( i=0 ; i < 10; i++ )); do + working_state="$(chacha20_inner_block $working_state)" + done + read ws0 ws1 ws2 ws3 ws4 ws5 ws6 ws7 ws8 ws9 ws10 ws11 ws12 ws13 ws14 ws15 <<< "$working_state" + + # Add working state to state + s0="$(printf "%08X" $(((0x$s0+0x$ws0) & 0xffffffff)))" + s1="$(printf "%08X" $(((0x$s1+0x$ws1) & 0xffffffff)))" + s2="$(printf "%08X" $(((0x$s2+0x$ws2) & 0xffffffff)))" + s3="$(printf "%08X" $(((0x$s3+0x$ws3) & 0xffffffff)))" + s4="$(printf "%08X" $(((0x$s4+0x$ws4) & 0xffffffff)))" + s5="$(printf "%08X" $(((0x$s5+0x$ws5) & 0xffffffff)))" + s6="$(printf "%08X" $(((0x$s6+0x$ws6) & 0xffffffff)))" + s7="$(printf "%08X" $(((0x$s7+0x$ws7) & 0xffffffff)))" + s8="$(printf "%08X" $(((0x$s8+0x$ws8) & 0xffffffff)))" + s9="$(printf "%08X" $(((0x$s9+0x$ws9) & 0xffffffff)))" + s10="$(printf "%08X" $(((0x$s10+0x$ws10) & 0xffffffff)))" + s11="$(printf "%08X" $(((0x$s11+0x$ws11) & 0xffffffff)))" + s12="$(printf "%08X" $(((0x$s12+0x$ws12) & 0xffffffff)))" + s13="$(printf "%08X" $(((0x$s13+0x$ws13) & 0xffffffff)))" + s14="$(printf "%08X" $(((0x$s14+0x$ws14) & 0xffffffff)))" + s15="$(printf "%08X" $(((0x$s15+0x$ws15) & 0xffffffff)))" + + # serialize the state + s0="${s0:6:2}${s0:4:2}${s0:2:2}${s0:0:2}" + s1="${s1:6:2}${s1:4:2}${s1:2:2}${s1:0:2}" + s2="${s2:6:2}${s2:4:2}${s2:2:2}${s2:0:2}" + s3="${s3:6:2}${s3:4:2}${s3:2:2}${s3:0:2}" + s4="${s4:6:2}${s4:4:2}${s4:2:2}${s4:0:2}" + s5="${s5:6:2}${s5:4:2}${s5:2:2}${s5:0:2}" + s6="${s6:6:2}${s6:4:2}${s6:2:2}${s6:0:2}" + s7="${s7:6:2}${s7:4:2}${s7:2:2}${s7:0:2}" + s8="${s8:6:2}${s8:4:2}${s8:2:2}${s8:0:2}" + s9="${s9:6:2}${s9:4:2}${s9:2:2}${s9:0:2}" + s10="${s10:6:2}${s10:4:2}${s10:2:2}${s10:0:2}" + s11="${s11:6:2}${s11:4:2}${s11:2:2}${s11:0:2}" + s12="${s12:6:2}${s12:4:2}${s12:2:2}${s12:0:2}" + s13="${s13:6:2}${s13:4:2}${s13:2:2}${s13:0:2}" + s14="${s14:6:2}${s14:4:2}${s14:2:2}${s14:0:2}" + s15="${s15:6:2}${s15:4:2}${s15:2:2}${s15:0:2}" + + tm_out "$s0$s1$s2$s3$s4$s5$s6$s7$s8$s9$s10$s11$s12$s13$s14$s15" + return 0 +} + +# See RFC 7539, Section 2.4 +chacha20() { + local key="$1" + local -i counter=1 + local nonce="$2" + local ciphertext="$3" + local -i i ciphertext_len num_blocks mod_check + local -i i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 i16 + local keystream plaintext="" + + ciphertext_len=${#ciphertext} + num_blocks=$ciphertext_len/128 + + for (( i=0; i < num_blocks; i++)); do + i1=128*$i; i2=$i1+8; i3=$i1+16; i4=$i1+24; i5=$i1+32; i6=$i1+40; i7=$i1+48; i8=$i1+56 + i9=$i1+64; i10=$i1+72; i11=$i1+80; i12=$i1+88; i13=$i1+96; i14=$i1+104; i15=$i1+112; i16=$i1+120 + keystream="$(chacha20_block "$key" "$(printf "%08X" $counter)" "$nonce")" + plaintext+="$(printf "%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X" \ + "$((0x${ciphertext:i1:8} ^ 0x${keystream:0:8}))" \ + "$((0x${ciphertext:i2:8} ^ 0x${keystream:8:8}))" \ + "$((0x${ciphertext:i3:8} ^ 0x${keystream:16:8}))" \ + "$((0x${ciphertext:i4:8} ^ 0x${keystream:24:8}))" \ + "$((0x${ciphertext:i5:8} ^ 0x${keystream:32:8}))" \ + "$((0x${ciphertext:i6:8} ^ 0x${keystream:40:8}))" \ + "$((0x${ciphertext:i7:8} ^ 0x${keystream:48:8}))" \ + "$((0x${ciphertext:i8:8} ^ 0x${keystream:56:8}))" \ + "$((0x${ciphertext:i9:8} ^ 0x${keystream:64:8}))" \ + "$((0x${ciphertext:i10:8} ^ 0x${keystream:72:8}))" \ + "$((0x${ciphertext:i11:8} ^ 0x${keystream:80:8}))" \ + "$((0x${ciphertext:i12:8} ^ 0x${keystream:88:8}))" \ + "$((0x${ciphertext:i13:8} ^ 0x${keystream:96:8}))" \ + "$((0x${ciphertext:i14:8} ^ 0x${keystream:104:8}))" \ + "$((0x${ciphertext:i15:8} ^ 0x${keystream:112:8}))" \ + "$((0x${ciphertext:i16:8} ^ 0x${keystream:120:8}))")" + counter+=1 + done + + mod_check=$ciphertext_len%128 + if [[ $mod_check -ne 0 ]]; then + keystream="$(chacha20_block "$key" "$(printf "%08X" $counter)" "$nonce")" + i1=128*$num_blocks + for (( i=0; i < mod_check; i=i+2 )); do + plaintext+="$(printf "%02X" "$((0x${ciphertext:i1:2} ^ 0x${keystream:i:2}))")" + i1+=2 + done + fi + tm_out "$plaintext" + return 0 +} + +# arg1: TLS cipher +# arg2: key +# arg3: nonce (must be 96 bits in length) +# arg4: ciphertext +sym-decrypt() { + local cipher="$1" + local key="$2" nonce="$3" + local ciphertext="$4" + local ossl_cipher + local plaintext + local -i ciphertext_len tag_len + + case "$cipher" in + *CCM_8*) + tag_len=16 ;; + *CCM*|*GCM*|*CHACHA20_POLY1305*) + tag_len=32 ;; + *) + return 7 ;; + esac + + # The final $tag_len characters of the ciphertext are the authentication tag + ciphertext_len=${#ciphertext} + [[ $ciphertext_len -lt $tag_len ]] && return 7 + ciphertext_len=$ciphertext_len-$tag_len + + if [[ "$cipher" =~ CHACHA20_POLY1305 ]]; then + if "$HAS_CHACHA20"; then + plaintext="$(asciihex_to_binary_file "${ciphertext:0:ciphertext_len}" "/dev/stdout" | \ + $OPENSSL enc -chacha20 -K "$key" -iv "01000000$nonce" 2>/dev/null | hexdump -v -e '16/1 "%02X"')" + plaintext="$(strip_spaces "$plaintext")" + else + plaintext="$(chacha20 "$key" "$nonce" "${ciphertext:0:ciphertext_len}")" + fi + elif [[ "$cipher" == "TLS_AES_128_GCM_SHA256" ]] && "$HAS_AES128_GCM"; then + plaintext="$(asciihex_to_binary_file "${ciphertext:0:ciphertext_len}" "/dev/stdout" | \ + $OPENSSL enc -aes-128-gcm -K "$key" -iv "$nonce" 2>/dev/null | hexdump -v -e '16/1 "%02X"')" + plaintext="$(strip_spaces "$plaintext")" + elif [[ "$cipher" == "TLS_AES_256_GCM_SHA384" ]] && "$HAS_AES256_GCM"; then + plaintext="$(asciihex_to_binary_file "${ciphertext:0:ciphertext_len}" "/dev/stdout" | \ + $OPENSSL enc -aes-256-gcm -K "$key" -iv "$nonce" 2>/dev/null | hexdump -v -e '16/1 "%02X"')" + plaintext="$(strip_spaces "$plaintext")" + else + if [[ "$cipher" =~ AES_128 ]]; then + ossl_cipher="-aes-128-ecb" + elif [[ "$cipher" =~ AES_256 ]]; then + ossl_cipher="-aes-256-ecb" + else + return 7 + fi + if [[ "$cipher" =~ CCM ]]; then + plaintext="$(ccm-gcm-decrypt "$ossl_cipher" "$key" "02${nonce}000001" "${ciphertext:0:ciphertext_len}")" + else # GCM + plaintext="$(ccm-gcm-decrypt "$ossl_cipher" "$key" "${nonce}00000002" "${ciphertext:0:ciphertext_len}")" + fi + fi + [[ $? -ne 0 ]] && return 7 + + tm_out "$plaintext" + return 0 +} + +# arg1: iv +# arg2: sequence number +get-nonce() { + local iv="$1" + local -i seq_num="$2" + local -i len lsb + local msb nonce + + len=${#iv} + [[ $len -lt 8 ]] && return 7 + i=$len-8 + msb="${iv:0:i}" + lsb="0x${iv:i:8}" + nonce="${msb}$(printf "%08X" "$(($lsb ^ $seq_num))")" + tm_out "$nonce" + return 0 +} + # Return: # 0 if arg1 contains the entire server response. # 1 if arg1 does not contain the entire server response. # 2 if the response is malformed. # 3 if (a) the response version is TLSv1.3; -# (b) arg1 contains the entire ServerHello (and appears to contain the entire response); and -# (c) the entire response is supposed to be parsed +# (b) arg1 contains the entire ServerHello (and appears to contain the entire response); +# (c) the entire response is supposed to be parsed; and +# (d) the key and IV have not been provided to decrypt the response. # arg1: ASCII-HEX encoded reply # arg2: whether to process the full request ("all") or just the basic request plus the ephemeral key if any ("ephemeralkey"). +# arg3: TLS cipher for decrypting TLSv1.3 response +# arg4: key and IV for decrypting TLSv1.3 response check_tls_serverhellodone() { local tls_hello_ascii="$1" local process_full="$2" + local cipher="$3" + local key_and_iv="$4" local tls_handshake_ascii="" tls_alert_ascii="" local -i i tls_hello_ascii_len tls_handshake_ascii_len tls_alert_ascii_len local -i msg_len remaining tls_serverhello_ascii_len sid_len local -i j offset tls_extensions_len extension_len local tls_content_type tls_protocol tls_handshake_type tls_msg_type extension_type local tls_err_level + local key iv + local -i seq_num=0 plaintext_len + local plaintext decrypted_response="" DETECTED_TLS_VERSION="" + [[ -n "$key_and_iv" ]] && read key iv <<< "$key_and_iv" + if [[ -z "$tls_hello_ascii" ]]; then return 0 # no server hello received fi @@ -8788,6 +9139,7 @@ check_tls_serverhellodone() { if [[ "$tls_content_type" == "16" ]]; then tls_handshake_ascii+="${tls_hello_ascii:i:msg_len}" tls_handshake_ascii_len=${#tls_handshake_ascii} + decrypted_response+="$tls_content_type$tls_protocol$(printf "%04X" $(($msg_len/2)))${tls_hello_ascii:i:msg_len}" # the ServerHello MUST be the first handshake message [[ $tls_handshake_ascii_len -ge 2 ]] && [[ "${tls_handshake_ascii:0:2}" != "02" ]] && return 2 if [[ $tls_handshake_ascii_len -ge 12 ]]; then @@ -8825,12 +9177,35 @@ check_tls_serverhellodone() { if [[ 0x$DETECTED_TLS_VERSION -ge 0x0304 ]] && [[ "$process_full" == "ephemeralkey" ]]; then tls_serverhello_ascii_len=2*$(hex2dec "${tls_handshake_ascii:2:6}") if [[ $tls_handshake_ascii_len -ge $tls_serverhello_ascii_len+8 ]]; then + tm_out "" return 0 # The entire ServerHello message has been received (and the rest isn't needed) fi fi fi elif [[ "$tls_content_type" == "15" ]]; then # TLS ALERT tls_alert_ascii+="${tls_hello_ascii:i:msg_len}" + decrypted_response+="$tls_content_type$tls_protocol$(printf "%04X" $(($msg_len/2)))${tls_hello_ascii:i:msg_len}" + elif [[ "$tls_content_type" == "17" ]] && [[ -n "$key_and_iv" ]]; then # encrypted data + nonce="$(get-nonce "$iv" "$seq_num")" + [[ $? -ne 0 ]] && return 2 + plaintext="$(sym-decrypt "$cipher" "$key" "$nonce" "${tls_hello_ascii:i:msg_len}")" + [[ $? -ne 0 ]] && return 2 + seq_num+=1 + + # Remove zeros from end of plaintext, if any + plaintext_len=${#plaintext}-2 + while [[ "${plaintext:plaintext_len:2}" == "00" ]]; do + plaintext_len=$plaintext_len-2 + done + tls_content_type="${plaintext:plaintext_len:2}" + decrypted_response+="${tls_content_type}0301$(printf "%04X" $(($plaintext_len/2)))${plaintext:0:plaintext_len}" + if [[ "$tls_content_type" == "16" ]]; then + tls_handshake_ascii+="${plaintext:0:plaintext_len}" + elif [[ "$tls_content_type" == "15" ]]; then + tls_alert_ascii+="${plaintext:0:plaintext_len}" + else + return 2 + fi fi done @@ -8840,7 +9215,7 @@ check_tls_serverhellodone() { remaining=$tls_alert_ascii_len-$i [[ $remaining -lt 4 ]] && return 1 tls_err_level=${tls_alert_ascii:i:2} # 1: warning, 2: fatal - [[ $tls_err_level == "02" ]] && DETECTED_TLS_VERSION="" && return 0 + [[ $tls_err_level == "02" ]] && DETECTED_TLS_VERSION="" && tm_out "" && return 0 done # If there is a serverHelloDone or Finished, then we are done. @@ -8857,14 +9232,15 @@ check_tls_serverhellodone() { # For SSLv3 - TLS1.2 look for a ServerHelloDone message. # For TLS 1.3 look for a Finished message. - [[ $tls_msg_type == "0E" ]] && return 0 - [[ $tls_msg_type == "14" ]] && return 0 + [[ $tls_msg_type == "0E" ]] && tm_out "" && return 0 + [[ $tls_msg_type == "14" ]] && tm_out "$decrypted_response" && return 0 done - # If the response is TLSv1.3 and the full response is to be processed, - # then return 3 if the entire ServerHello has been received. + # If the response is TLSv1.3 and the full response is to be processed, but the + # key and IV have not been provided to decrypt the response, then return 3 if + # the entire ServerHello has been received. if [[ "$DETECTED_TLS_VERSION" == "0304" ]] && [[ "$process_full" == "all" ]] && \ - [[ $tls_handshake_ascii_len -gt 0 ]]; then - return 3 + [[ -z "$key_and_iv" ]] && [[ $tls_handshake_ascii_len -gt 0 ]]; then + return 3 fi # If we haven't encoountered a fatal alert or a server hello done, # then there must be more data to retrieve. @@ -8931,14 +9307,17 @@ parse_tls_serverhello() { local -i tls_hello_ascii_len tls_handshake_ascii_len tls_alert_ascii_len msg_len local tls_serverhello_ascii="" tls_certificate_ascii="" local tls_serverkeyexchange_ascii="" tls_certificate_status_ascii="" + local tls_encryptedextensions_ascii="" tls_revised_certificate_msg="" local -i tls_serverhello_ascii_len=0 tls_certificate_ascii_len=0 local -i tls_serverkeyexchange_ascii_len=0 tls_certificate_status_ascii_len=0 + local -i tls_encryptedextensions_ascii_len=0 + local added_encrypted_extensions=false local tls_alert_descrip tls_sid_len_hex issuerDN subjectDN CAissuerDN CAsubjectDN local -i tls_sid_len offset extns_offset nr_certs=0 local tls_msg_type tls_content_type tls_protocol tls_protocol2 tls_hello_time local tls_err_level tls_err_descr_no tls_cipher_suite rfc_cipher_suite tls_compression_method local tls_extensions="" extension_type named_curve_str="" named_curve_oid - local -i i j extension_len tls_extensions_len ocsp_response_len=0 ocsp_response_list_len ocsp_resp_offset + local -i i j extension_len extn_len tls_extensions_len ocsp_response_len=0 ocsp_response_list_len ocsp_resp_offset local -i certificate_list_len certificate_len cipherlist_len local -i curve_type named_curve local -i dh_bits=0 msb mask @@ -9149,6 +9528,14 @@ parse_tls_serverhello() { fi tls_serverhello_ascii="${tls_handshake_ascii:i:msg_len}" tls_serverhello_ascii_len=$msg_len + elif [[ "$process_full" == "all" ]] && [[ "$tls_msg_type" == "08" ]]; then + # Add excrypted extensions (now decrypted) to end of extensions in SeverHello + tls_encryptedextensions_ascii="${tls_handshake_ascii:i:msg_len}" + tls_encryptedextensions_ascii_len=$msg_len + if [[ $msg_len -lt 2 ]]; then + debugme tmln_warning "Response contained a malformed encrypted extensions message" + return 1 + fi elif [[ "$process_full" == "all" ]] && [[ "$tls_msg_type" == "0B" ]]; then if [[ -n "$tls_certificate_ascii" ]]; then debugme tmln_warning "Response contained more than one Certificate handshake message." @@ -9450,6 +9837,74 @@ parse_tls_serverhello() { FF01) tls_extensions+="TLS server extension \"renegotiation info\" (id=65281), len=$extension_len\n" ;; *) tls_extensions+="TLS server extension \"unrecognized extension\" (id=$(printf "%d\n\n" "0x$extension_type")), len=$extension_len\n" ;; esac + # After processing all of the extensions in the ServerHello message, + # if it has been determined that the response is TLSv1.3 and the + # response was decrypted, then modify $tls_serverhello_ascii by adding + # the extensions from the EncryptedExtensions and Certificate messages + # and then process them. + if ! "$added_encrypted_extensions" && [[ "$DETECTED_TLS_VERSION" == "0304" ]] && \ + [[ $((i+8+extension_len)) -eq $tls_extensions_len ]]; then + # Note that the encrypted extensions have been added so that + # the aren't added a second time. + added_encrypted_extensions=true + if [[ -n "$tls_encryptedextensions_ascii" ]]; then + tls_serverhello_ascii_len+=$tls_encryptedextensions_ascii_len-4 + tls_extensions_len+=$tls_encryptedextensions_ascii_len-4 + tls_encryptedextensions_ascii_len=$tls_encryptedextensions_ascii_len/2-2 + let offset=$extns_offset+4 + tls_serverhello_ascii="${tls_serverhello_ascii:0:extns_offset}$(printf "%04X" $((0x${tls_serverhello_ascii:extns_offset:4}+$tls_encryptedextensions_ascii_len)))${tls_serverhello_ascii:offset}${tls_encryptedextensions_ascii:4}" + fi + if [[ -n "$tls_certificate_ascii" ]]; then + # In TLS 1.3, the Certificate message begins with a zero length certificate_request_context. + # In addition, certificate_list is now a list of (certificate, extension) pairs rather than + # just certificates. So, extract the extensions and add them to $tls_serverhello_ascii and + # create a new $tls_certificate_ascii that only contains a list of certificates. + if [[ -n "$tls_certificate_ascii" ]]; then + if [[ "${tls_certificate_ascii:0:2}" != "00" ]]; then + debugme tmln_warning "Malformed Certificate Handshake message in ServerHello." + tmpfile_handle $FUNCNAME.txt + return 1 + fi + if [[ $tls_certificate_ascii_len -lt 8 ]]; then + debugme tmln_warning "Malformed Certificate Handshake message in ServerHello." + tmpfile_handle $FUNCNAME.txt + return 1 + fi + certificate_list_len=2*$(hex2dec "${tls_certificate_ascii:2:6}") + if [[ $certificate_list_len -ne $tls_certificate_ascii_len-8 ]]; then + debugme tmln_warning "Malformed Certificate Handshake message in ServerHello." + tmpfile_handle $FUNCNAME.txt + return 1 + fi + + for (( j=8; j < tls_certificate_ascii_len; j=j+extn_len )); do + if [[ $tls_certificate_ascii_len-$j -lt 6 ]]; then + debugme tmln_warning "Malformed Certificate Handshake message in ServerHello." + tmpfile_handle $FUNCNAME.txt + return 1 + fi + certificate_len=2*$(hex2dec "${tls_certificate_ascii:j:6}") + if [[ $certificate_len -gt $tls_certificate_ascii_len-$j-6 ]]; then + debugme tmln_warning "Malformed Certificate Handshake message in ServerHello." + tmpfile_handle $FUNCNAME.txt + return 1 + fi + len1=$certificate_len+6 + tls_revised_certificate_msg+="${tls_certificate_ascii:j:len1}" + j+=$len1 + extn_len=2*$(hex2dec "${tls_certificate_ascii:j:4}") + j+=4 + # TODO: Should only the extensions associated with the EE certificate be added to $tls_serverhello_ascii? + tls_serverhello_ascii_len+=$extn_len + tls_extensions_len+=$extn_len + let offset=$extns_offset+4 + tls_serverhello_ascii="${tls_serverhello_ascii:0:extns_offset}$(printf "%04X" $((0x${tls_serverhello_ascii:extns_offset:4}+$extn_len/2)))${tls_serverhello_ascii:offset}${tls_certificate_ascii:j:extn_len}" + done + tls_certificate_ascii_len=${#tls_revised_certificate_msg}+6 + tls_certificate_ascii="$(printf "%06X" $(($tls_certificate_ascii_len/2-3)))$tls_revised_certificate_msg" + fi + fi + fi done fi @@ -10602,7 +11057,7 @@ tls_sockets() { local process_full="$3" offer_compression=false skip=false local close_connection=true local -i hello_done=0 - local cipher="" key_and_iv="" + local cipher="" key_and_iv="" decrypted_response [[ "$5" == "true" ]] && offer_compression=true [[ "$6" == "false" ]] && close_connection=false @@ -10690,9 +11145,11 @@ tls_sockets() { fi skip=false if [[ $hello_done -eq 1 ]]; then - check_tls_serverhellodone "$tls_hello_ascii" "$process_full" + decrypted_response="$(check_tls_serverhellodone "$tls_hello_ascii" "$process_full" "$cipher" "$key_and_iv")" hello_done=$? + [[ "$hello_done" -eq 0 ]] && [[ -n "$decrypted_response" ]] && tls_hello_ascii="$(toupper "$decrypted_response")" if [[ "$hello_done" -eq 3 ]]; then + hello_done=1; skip=true debugme echo "reading server hello..." parse_tls_serverhello "$tls_hello_ascii" "ephemeralkey" ret=$? @@ -10707,13 +11164,6 @@ tls_sockets() { else hello_done=2 fi - # The following three lines are temporary until the code - # to decrypt TLSv1.3 responses has been added, at which point - # parse_tls_serverhello() will be called with process_full="all" - # and parse_tls_serverhello() will populate these files. - process_full="ephemeralkey" - [[ -e "$HOSTCERT" ]] && rm "$HOSTCERT" - [[ -e "$TEMPDIR/intermediatecerts.pem" ]] && rm "$TEMPDIR/intermediatecerts.pem" fi fi done @@ -13609,6 +14059,15 @@ find_openssl_binary() { grep -q 'mysql' $s_client_starttls_has && \ HAS_MYSQL=true + $OPENSSL enc -chacha20 -K "12345678901234567890123456789012" -iv "01000000123456789012345678901234" > /dev/null 2> /dev/null <<< "test" + [[ $? -eq 0 ]] && HAS_CHACHA20=true + + $OPENSSL enc -aes-128-gcm -K 0123456789abcdef0123456789abcdef -iv 0123456789abcdef01234567 > /dev/null 2> /dev/null <<< "test" + [[ $? -eq 0 ]] && HAS_AES128_GCM=true + + $OPENSSL enc -aes-256-gcm -K 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef -iv 0123456789abcdef01234567 > /dev/null 2> /dev/null <<< "test" + [[ $? -eq 0 ]] && HAS_AES256_GCM=true + if [[ "$OPENSSL_TIMEOUT" != "" ]]; then if type -p timeout 2>&1 >/dev/null ; then if ! "$do_mass_testing"; then From 1488baeac5deaa56f7f2ad4697c0540c7db4c7db Mon Sep 17 00:00:00 2001 From: Dirk Date: Wed, 20 Dec 2017 09:00:00 +0100 Subject: [PATCH 3/8] Documentation of CA_BUNDLES_PATH See also #941 --- doc/testssl.1 | 3 +++ doc/testssl.1.html | 5 ++++- doc/testssl.1.md | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/testssl.1 b/doc/testssl.1 index 133af5c..908780e 100644 --- a/doc/testssl.1 +++ b/doc/testssl.1 @@ -498,6 +498,9 @@ HEARTBLEED_MAX_WAITSOCK Is the similar to MAX_WAITSOCK but applies only to the S .IP "\(bu" 4 MEASURE_TIME_FILE For seldom cases when you don\'t want the scan time to be included in the output you can set this to false\. . +.IP "\(bu" 4 +CA_BUNDLES_PATH: If you have an own set of CA bundles or you want to point testssl\.sh to a specific location of a CA bundle, you can use this variable to set the directory which testssl\.sh will use\. Please note that it overrides completely the builtin path of testssl\.sh which means that you will only test against the bundles you point to\. Also you might want to use ~/utils/create_ca_hashes\.sh to create the hashes for HPKP\. +. .IP "" 0 . .SH "EXAMPLES" diff --git a/doc/testssl.1.html b/doc/testssl.1.html index d403115..2a7fff8 100644 --- a/doc/testssl.1.html +++ b/doc/testssl.1.html @@ -403,7 +403,10 @@ The same can be achieved by setting the environment variable WARNINGSMAX_WAITSOCK: It instructs testssl.sh to wait until the specified time before declaring a socket connection dead. Don't change this unless you're absolutely sure what you're doing. Value is in seconds.
  • CCS_MAX_WAITSOCK Is the similar to above but applies only to the CCS handshakes, for both of the two the two CCS payload. Don't change this unless you're absolutely sure what you're doing. Value is in seconds.
  • HEARTBLEED_MAX_WAITSOCK Is the similar to MAX_WAITSOCK but applies only to the ServerHello after sending the Heartbleed payload. Don't change this unless you're absolutely sure what you're doing. Value is in seconds.
  • -
  • MEASURE_TIME_FILE For seldom cases when you don't want the scan time to be included in the output you can set this to false.
  • +
  • MEASURE_TIME_FILE For seldom cases when you don't want the scan time to be included in the output you can set this to false.

  • +
  • CA_BUNDLES_PATH: If you have an own set of CA bundles or you want to point testssl.sh to a specific location of a CA bundle, you can use this variable to set the directory which testssl.sh will +use. Please note that it overrides completely the builtin path of testssl.sh which means that you will only test against the bundles you point to. Also you might want to use ~/utils/create_ca_hashes.sh +to create the hashes for HPKP.

  • diff --git a/doc/testssl.1.md b/doc/testssl.1.md index 88a9c53..1d50da4 100644 --- a/doc/testssl.1.md +++ b/doc/testssl.1.md @@ -334,7 +334,10 @@ Except the environment variables mentioned above which replace command line opti [comment]: # DAYS2WARN1 [comment]: # DAYS2WARN2 [comment]: # TESTSSL_INSTALL_DIR -[comment]: # CA_BUNDLES_PATH +* CA_BUNDLES_PATH: If you have an own set of CA bundles or you want to point testssl.sh to a specific location of a CA bundle, you can use this variable to set the directory which testssl.sh will +use. Please note that it overrides completely the builtin path of testssl.sh which means that you will only test against the bundles you point to. Also you might want to use ~/utils/create_ca_hashes.sh +to create the hashes for HPKP. + [comment]: # CAPATH From 65e435eb704dbd8f42a611f7161634e16e694b9e Mon Sep 17 00:00:00 2001 From: David Cooper Date: Wed, 20 Dec 2017 10:17:21 -0500 Subject: [PATCH 4/8] Process TLSv1.3 status_request extension In TLSv1.2 and below, servers respond to a status_request extension (a request for a stapled OCSP response) by returning an empty status_request extension and then including a CertificateStatus message, which follows the Certificate message. In TLSv1.3 the CertificateStatus response is included as the value of the status_request extension, which now appears as an extension within the Certificate message. This PR extracts the contents of the status_request extension sent by the server so that it can later be processed in the same way as if it had sent in a TLSv1.2 or below response. --- testssl.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/testssl.sh b/testssl.sh index 00e248a..17b7d34 100755 --- a/testssl.sh +++ b/testssl.sh @@ -9663,7 +9663,17 @@ parse_tls_serverhello() { 0002) tls_extensions+="TLS server extension \"client certificate URL\" (id=2), len=$extension_len\n" ;; 0003) tls_extensions+="TLS server extension \"trusted CA keys\" (id=3, len=$extension_len\n)" ;; 0004) tls_extensions+="TLS server extension \"truncated HMAC\" (id=4), len=$extension_len\n" ;; - 0005) tls_extensions+="TLS server extension \"status request\" (id=5), len=$extension_len\n" ;; + 0005) tls_extensions+="TLS server extension \"status request\" (id=5), len=$extension_len\n" + if [[ $extension_len -gt 0 ]] && [[ "$process_full" == "all" ]]; then + # In TLSv1.3 the status_request extension contains the CertificateStatus message, unlike + # TLSv1.2 and below where CertificateStatus appears in its own handshake message. So, if + # the status_request extension is not empty, extract the value and place it in + # $tls_certificate_status_ascii. + tls_certificate_status_ascii_len=$extension_len + let offset=$extns_offset+12+$i + tls_certificate_status_ascii="${tls_serverhello_ascii:offset:tls_certificate_status_ascii_len}" + fi + ;; 0006) tls_extensions+="TLS server extension \"user mapping\" (id=6), len=$extension_len\n" ;; 0007) tls_extensions+="TLS server extension \"client authz\" (id=7), len=$extension_len\n" ;; 0008) tls_extensions+="TLS server extension \"server authz\" (id=8), len=$extension_len\n" ;; From 14908bac98a08b309cb28c258aef31e10c6540c7 Mon Sep 17 00:00:00 2001 From: David Cooper Date: Wed, 20 Dec 2017 10:40:17 -0500 Subject: [PATCH 5/8] Process supported_groups extension In TLSv1.3 servers may send a supported_groups extension, which "SHOULD contain all groups the server supports, regardless of whether they are currently supported by the client." This PR extracts the contents of the supported_groups extension, if `parse_tls_serverhello()` is to process "all" of the server's response. The contents of the extension are also displayed on the terminal if $DEBUG -ge 3. --- testssl.sh | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/testssl.sh b/testssl.sh index 00e248a..2a6d6cb 100755 --- a/testssl.sh +++ b/testssl.sh @@ -9668,7 +9668,40 @@ parse_tls_serverhello() { 0007) tls_extensions+="TLS server extension \"client authz\" (id=7), len=$extension_len\n" ;; 0008) tls_extensions+="TLS server extension \"server authz\" (id=8), len=$extension_len\n" ;; 0009) tls_extensions+="TLS server extension \"cert type\" (id=9), len=$extension_len\n" ;; - 000A) tls_extensions+="TLS server extension \"supported_groups\" (id=10), len=$extension_len\n" ;; + 000A) tls_extensions+="TLS server extension \"supported_groups\" (id=10), len=$extension_len\n" + if [[ "$process_full" == "all" ]]; then + if [[ $extension_len -lt 4 ]]; then + debugme tmln_warning "Malformed supported groups extension." + return 1 + fi + echo -n "Supported groups: " >> $TMPFILE + let offset=$extns_offset+12+$i + len1=2*$(hex2dec "${tls_serverhello_ascii:offset:4}") + if [[ $extension_len -lt $len1+4 ]] || [[ $len1 -lt 4 ]]; then + debugme tmln_warning "Malformed supported groups extension." + return 1 + fi + let offset=$offset+4 + for (( j=0; j < len1; j=j+4 )); do + [[ $j -ne 0 ]] && echo -n ", " >> $TMPFILE + case "${tls_serverhello_ascii:offset:4}" in + "0017") echo -n "secp256r1" >> $TMPFILE ;; + "0018") echo -n "secp384r1" >> $TMPFILE ;; + "0019") echo -n "secp521r1" >> $TMPFILE ;; + "001D") echo -n "X25519" >> $TMPFILE ;; + "001E") echo -n "X448" >> $TMPFILE ;; + "0100") echo -n "ffdhe2048" >> $TMPFILE ;; + "0101") echo -n "ffdhe3072" >> $TMPFILE ;; + "0102") echo -n "ffdhe4096" >> $TMPFILE ;; + "0103") echo -n "ffdhe6144" >> $TMPFILE ;; + "0104") echo -n "ffdhe8192" >> $TMPFILE ;; + *) echo -n "unknown (${tls_serverhello_ascii:offset:4})" >> $TMPFILE ;; + esac + let offset=$offset+4 + done + echo "" >> $TMPFILE + fi + ;; 000B) tls_extensions+="TLS server extension \"EC point formats\" (id=11), len=$extension_len\n" ;; 000C) tls_extensions+="TLS server extension \"SRP\" (id=12), len=$extension_len\n" ;; 000D) tls_extensions+="TLS server extension \"signature algorithms\" (id=13), len=$extension_len\n" ;; @@ -9991,6 +10024,9 @@ parse_tls_serverhello() { -e 's/,.*$/,/g' -e 's/),$/\"/g' \ -e 's/elliptic curves\/#10/supported_groups\/#10/g')" echo "" + if [[ "$tls_extensions" =~ "supported_groups" ]]; then + echo " Supported Groups: $(grep "Supported groups:" "$TMPFILE" | sed 's/Supported groups: //')" + fi if [[ "$tls_extensions" =~ "application layer protocol negotiation" ]]; then echo " ALPN protocol: $(grep "ALPN protocol:" "$TMPFILE" | sed 's/ALPN protocol: //')" fi From 5c005ac139dce974e4bdf91a34812cfed9d90283 Mon Sep 17 00:00:00 2001 From: Dirk Date: Wed, 20 Dec 2017 19:21:33 +0100 Subject: [PATCH 6/8] Add '--full' / '-9' ... .. to check during the default run for server implemenation bugs and run cipher per procol check instead of cipher check. Please not that this option could disappear later. --- testssl.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/testssl.sh b/testssl.sh index 88d570b..d70368f 100755 --- a/testssl.sh +++ b/testssl.sh @@ -14210,7 +14210,7 @@ help() { Alternatively: nmap output in greppable format (-oG) (1x port per line allowed) --mode Mass testing to be done serial (default) or parallel (--parallel is shortcut for the latter) -single check as ("$PROG_NAME URI" does everything except -E and -g): +single check as ("$PROG_NAME URI" does everything except -E and -g): -e, --each-cipher checks each local cipher remotely -E, --cipher-per-proto checks those per protocol -s, --std, --standard tests certain lists of cipher suites by strength @@ -14241,10 +14241,11 @@ single check as ("$PROG_NAME URI" does everything except -E and -g): -f, --pfs, --fs, --nsa checks (perfect) forward secrecy settings -4, --rc4, --appelbaum which RC4 ciphers are being offered? -g, --grease tests several server implementation bugs like GREASE and size limitations + -9, --full includes tests for implementation bugs and cipher per protocol (could disappear) tuning / connect options (most also can be preset via environment variables): --fast omits some checks: using openssl for all ciphers (-e), show only first - preferred cipher + preferred cipher. --bugs enables the "-bugs" option of s_client, needed e.g. for some buggy F5s --assume-http if protocol check fails it assumes HTTP protocol and enforces HTTP checks --ssl-native fallback to checks with OpenSSL where sockets are normally used @@ -16074,6 +16075,12 @@ parse_cmd_line() { -g|--grease) do_grease=true ;; + -9|--full) + set_scanning_defaults + do_allciphers=false + do_cipher_per_proto=true + do_grease=true + ;; --devel) ### this development feature will soon disappear HEX_CIPHER="$TLS12_CIPHER" # DEBUG=3 ./testssl.sh --devel 03 "cc, 13, c0, 13" google.de --> TLS 1.2, old CHACHA/POLY @@ -16383,6 +16390,8 @@ lets_roll() { run_spdy; ret=$(($? + ret)); time_right_align run_spdy; run_http2; ret=$(($? + ret)); time_right_align run_http2; } + fileout_section_header $section_number true && ((section_number++)) + "$do_grease" && { run_grease; ret=$(($? + ret)); time_right_align run_grease; } fileout_section_header $section_number true && ((section_number++)) $do_std_cipherlists && { run_std_cipherlists; ret=$(($? + ret)); time_right_align run_std_cipherlists; } @@ -16446,8 +16455,6 @@ lets_roll() { fileout_section_header $section_number true && ((section_number++)) $do_client_simulation && { run_client_simulation; ret=$(($? + ret)); time_right_align run_client_simulation; } - fileout_section_header $section_number true && ((section_number++)) - "$do_grease" && { run_grease; ret=$(($? + ret)); time_right_align run_grease; } fileout_section_footer true outln From b9e67fcf29e46fbf2f62cf07f9c2096d2f73969e Mon Sep 17 00:00:00 2001 From: David Cooper Date: Wed, 20 Dec 2017 16:38:10 -0500 Subject: [PATCH 7/8] run_renego() and OpenSSL 1.1.1 run_renego() appears to produce a false positive if OpenSSL 1.1.1 is used and the server being tested supports TLSv1.3 (i.e., the server supports the same draft version of TLSv1.3 as the version of OpenSSL 1.1.1 being used does). This PR fixes the problem by telling calls to $OPENSSL s_client in run_renego() to not use TLSv1.3. --- testssl.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/testssl.sh b/testssl.sh index d70368f..9849424 100755 --- a/testssl.sh +++ b/testssl.sh @@ -11815,18 +11815,20 @@ run_ticketbleed() { run_renego() { # no SNI here. Not needed as there won't be two different SSL stacks for one IP - local legacycmd="" + local legacycmd="" proto="$OPTIMAL_PROTO" local insecure_renogo_str="Secure Renegotiation IS NOT" local sec_renego sec_client_renego local cve="CVE-2009-3555" local cwe="CWE-310" local hint="" + "$HAS_TLS13" && [[ -z "$proto" ]] && proto="-no_tls1_3" + [[ $VULN_COUNT -le $VULN_THRESHLD ]] && outln && pr_headlineln " Testing for Renegotiation vulnerabilities " && outln pr_bold " Secure Renegotiation "; out "($cve) " # and RFC 5746, OSVDB 59968-59974 # community.qualys.com/blogs/securitylabs/2009/11/05/ssl-and-tls-authentication-gap-vulnerability-discovered - $OPENSSL s_client $(s_client_options "$OPTIMAL_PROTO $STARTTLS $BUGS -connect $NODEIP:$PORT $SNI $PROXY") 2>&1 $TMPFILE 2>$ERRFILE + $OPENSSL s_client $(s_client_options "$proto $STARTTLS $BUGS -connect $NODEIP:$PORT $SNI $PROXY") 2>&1 $TMPFILE 2>$ERRFILE if sclient_connect_successful $? $TMPFILE; then grep -iaq "$insecure_renogo_str" $TMPFILE sec_renego=$? # 0= Secure Renegotiation IS NOT supported @@ -11865,7 +11867,7 @@ run_renego() { 1.0.1*|1.0.2*) legacycmd="-legacy_renegotiation" ;; - 0.9.9*|1.0*) + 0.9.9*|1.0*|1.1*) ;; # all ok esac @@ -11876,7 +11878,7 @@ run_renego() { else # We need up to two tries here, as some LiteSpeed servers don't answer on "R" and block. Thus first try in the background # msg enables us to look deeper into it while debugging - echo R | $OPENSSL s_client $(s_client_options "$OPTIMAL_PROTO $BUGS $legacycmd $STARTTLS -msg -connect $NODEIP:$PORT $SNI $PROXY") >$TMPFILE 2>>$ERRFILE & + echo R | $OPENSSL s_client $(s_client_options "$proto $BUGS $legacycmd $STARTTLS -msg -connect $NODEIP:$PORT $SNI $PROXY") >$TMPFILE 2>>$ERRFILE & wait_kill $! $HEADER_MAXSLEEP if [[ $? -eq 3 ]]; then pr_done_good "likely not vulnerable (OK)"; outln ", timed out" # it hung @@ -11884,7 +11886,7 @@ run_renego() { sec_client_renego=1 else # second try in the foreground as we are sure now it won't hang - echo R | $OPENSSL s_client $(s_client_options "$legacycmd $STARTTLS $BUGS -msg -connect $NODEIP:$PORT $SNI $PROXY") >$TMPFILE 2>>$ERRFILE + echo R | $OPENSSL s_client $(s_client_options "$proto $legacycmd $STARTTLS $BUGS -msg -connect $NODEIP:$PORT $SNI $PROXY") >$TMPFILE 2>>$ERRFILE sec_client_renego=$? # 0=client is renegotiating & doesn't return an error --> vuln! case "$sec_client_renego" in 0) if [[ $SERVICE == "HTTP" ]]; then From 978478fd0c08318a7235d2a8a26e4b5035956668 Mon Sep 17 00:00:00 2001 From: Dirk Date: Thu, 21 Dec 2017 15:06:08 +0100 Subject: [PATCH 8/8] Fix "typo" --- testssl.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/testssl.sh b/testssl.sh index 9849424..2a6df4f 100755 --- a/testssl.sh +++ b/testssl.sh @@ -14207,7 +14207,7 @@ help() { protocol is (latter 4 require supplied openssl) --xmpphost For STARTTLS enabled XMPP it supplies the XML stream to-'' domain -- sometimes needed --mx Tests MX records from high to low priority (STARTTLS, port 25) - --file Mass testing option: Reads command lines from , one line per instance. + --file Mass testing option: Reads command lines from , one line per instance. Comments via # allowed, EOF signals end of . Implicitly turns on "--warnings batch". Alternatively: nmap output in greppable format (-oG) (1x port per line allowed) --mode Mass testing to be done serial (default) or parallel (--parallel is shortcut for the latter) @@ -15495,7 +15495,7 @@ nmap_to_plain_file() { target_fname="${FNAME%.*}.txt" > "${target_fname}" if [[ $? -ne 0 ]]; then - # try to just create ${FNAME%.*}.txt in the same dir as the gmap file failed. + # try to just create ${FNAME%.*}.txt in the same dir as the gnmap file failed. # backup is using one in $TEMPDIR target_fname="${target_fname##*\/}" # strip path (Unix) target_fname="${target_fname##*\\}" # strip path (Dos) @@ -15539,7 +15539,7 @@ nmap_to_plain_file() { run_mass_testing() { local cmdline="" local first=true - local gmapadd="" + local gnmapadd="" local saved_fname="$FNAME" if [[ ! -r "$FNAME" ]] && "$IKNOW_FNAME"; then @@ -15547,11 +15547,11 @@ run_mass_testing() { fi if [[ "$(head -1 "$FNAME")" =~ (Nmap [4-8])(.*)( scan initiated )(.*) ]]; then - gmapadd="grep(p)able nmap " + gnmapadd="grep(p)able nmap " nmap_to_plain_file fi - pr_reverse "====== Running in file batch mode with ${gmapadd}file=\"$saved_fname\" ======"; outln "\n" + pr_reverse "====== Running in file batch mode with ${gnmapadd}file=\"$saved_fname\" ======"; outln "\n" while read cmdline; do cmdline="$(filter_input "$cmdline")" [[ -z "$cmdline" ]] && continue @@ -15612,7 +15612,7 @@ run_mass_testing_parallel() { local -i i nr_active_tests=0 local -a -i start_time=() local -i curr_time wait_time - local gmapadd="" + local gnmapadd="" local saved_fname="$FNAME" if [[ ! -r "$FNAME" ]] && $IKNOW_FNAME; then @@ -15620,11 +15620,11 @@ run_mass_testing_parallel() { fi if [[ "$(head -1 "$FNAME")" =~ (Nmap [4-8])(.*)( scan initiated )(.*) ]]; then - gmapadd="grep(p)able nmap " + gnmapadd="grep(p)able nmap " nmap_to_plain_file fi - pr_reverse "====== Running in file batch mode with ${gmapadd}file=\"$saved_fname\" ======"; outln "\n" + pr_reverse "====== Running in file batch mode with ${gnmapadd}file=\"$saved_fname\" ======"; outln "\n" while read cmdline; do cmdline="$(filter_input "$cmdline")" [[ -z "$cmdline" ]] && continue