From dd2409527c3669abd69a2f44b748639bf656e51a Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Tue, 29 Jul 2025 13:37:18 +0200 Subject: [PATCH] Start working on DNS HTTPS RR (RFC 9460) for 3.3dev This is a fresh start for #2484 as the PR wasn't ready yet for 3.2 by the time it was released. The info for the HTTPS RR shows up in the very beginning, i.e. in the service_detection(). All keys are listed now in bold, values in a regular font. `get_https_rrecord()` was introduced by copying and modifying `get_caa_rr_record()`. There's a similar obstacle as with CAA RRs: older binaries show the resource records binary encoded. Thus a new set of global vars is introduced HAS_*_HTTPS which check whether the binaries support decoding the RR directly. For CAA there was a minor bug fixed when records were queried also when it was instructed to minimize/skip or use proxy only. Todo: - Add logic in QUIC - if RR is detected and not QUIC is possible - add time for QUIC detection when RR is retrieved - show full HTTPS RR record, at least when having a new DNS client - shorten the comments in `get_https_rrecord()` - Man page - when ASSUME_HTTP is set and no services was detected: this needs to be handled --- testssl.sh | 310 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 267 insertions(+), 43 deletions(-) diff --git a/testssl.sh b/testssl.sh index 408a5fb..7d8e1e6 100755 --- a/testssl.sh +++ b/testssl.sh @@ -383,6 +383,11 @@ HAS_IDN2=false HAS_AVAHIRESOLVE=false HAS_DSCACHEUTIL=false HAS_DIG_NOIDNOUT=false +HAS_DIG_HTTPS=false # *_HTTPS: whether the binaries support HTTPS RR directly +HAS_DRILL_HTTPS=false +HAS_HOST_HTTPS=false +HAS_NSLOOKUP_HTTPS=false + HAS_XXD=false OSSL_CIPHERS_S="" @@ -2472,6 +2477,7 @@ s_client_options() { # determines whether the port has an HTTP service running or not (plain TLS, no STARTTLS) # arg1 could be the protocol determined as "working". IIS6 needs that. +# sets global $SERVICE # service_detection() { local -i was_killed @@ -2514,24 +2520,30 @@ service_detection() { debugme head -50 $TMPFILE | sed -e '//,$d' -e '//,$d' -e '/ trying HTTP checks" SERVICE=HTTP fileout "${jsonID}" "DEBUG" "Couldn't determine service -- ASSUME_HTTP set" elif [[ "$CLIENT_AUTH" == required ]] && [[ -z $MTLS ]]; then out " certificate-based authentication without providing client certificate and private key => skipping all HTTP checks" - echo "certificate-based authentication without providing client certificate and private key => skipping all HTTP checks" >$TMPFILE + echo "certificate-based authentication without providing client certificate and private key => skipping all HTTP checks" >$TMPFILE fileout "${jsonID}" "INFO" "certificate-based authentication without providing client certificate and private key => skipping all HTTP checks" else out " Couldn't determine what's running on port $PORT" @@ -2540,7 +2552,7 @@ service_detection() { out " -- ASSUME_HTTP set though" fileout "${jsonID}" "DEBUG" "Couldn't determine service -- ASSUME_HTTP set" else - out ", assuming no HTTP service => skipping all HTTP checks" + out ", assuming no HTTP => skipping all HTTP checks" fileout "${jsonID}" "DEBUG" "Couldn't determine service, skipping all HTTP checks" fi fi @@ -10134,40 +10146,42 @@ certificate_info() { must_staple "$json_postfix" "$provides_stapling" "$cert_txt" out "$indent"; pr_bold " DNS CAA RR"; out " (experimental) " - jsonID="DNS_CAArecord" + jsonID="DNS_CAA_rrecord" caa_node="$NODE" caa="" - while [[ -z "$caa" ]] && [[ -n "$caa_node" ]]; do - caa="$(get_caa_rr_record $caa_node)" - tmp=${PIPESTATUS[@]} - [[ $DEBUG -ge 4 ]] && echo "get_caa_rr_record: $tmp" - [[ $caa_node =~ '.'$ ]] || caa_node+="." - caa_node=${caa_node#*.} - done - if [[ -n "$caa" ]]; then - pr_svrty_good "available"; out " - please check for match with \"Issuer\" below" - if [[ $(count_lines "$caa") -eq 1 ]]; then - out ": " - else - outln; out "$spaces" - fi - while read caa; do - if [[ -n "$caa" ]]; then - all_caa+="$caa, " - fi - done <<< "$caa" - all_caa=${all_caa%, } # strip trailing comma - pr_italic "$(out_row_aligned_max_width "$all_caa" "$indent " $TERM_WIDTH)" - fileout "${jsonID}${json_postfix}" "OK" "$all_caa" - elif [[ -n "$NODNS" ]]; then + if [[ -n "$NODNS" ]]; then out "(instructed to minimize/skip DNS queries)" fileout "${jsonID}${json_postfix}" "INFO" "check skipped as instructed" elif "$DNS_VIA_PROXY"; then out "(instructed to use the proxy for DNS only)" fileout "${jsonID}${json_postfix}" "INFO" "check skipped as instructed (proxy)" else - pr_svrty_low "not offered" - fileout "${jsonID}${json_postfix}" "LOW" "--" + while [[ -z "$caa" ]] && [[ -n "$caa_node" ]]; do + caa="$(get_caa_rrecord $caa_node)" + tmp=${PIPESTATUS[@]} + [[ $DEBUG -ge 4 ]] && echo "get_https_caa_rr_record: $tmp" + [[ $caa_node =~ '.'$ ]] || caa_node+="." + caa_node=${caa_node#*.} + done + if [[ -n "$caa" ]]; then + pr_svrty_good "available"; out " - please check for match with \"Issuer\" below" + if [[ $(count_lines "$caa") -eq 1 ]]; then + out ": " + else + outln; out "$spaces" + fi + while read caa; do + if [[ -n "$caa" ]]; then + all_caa+="$caa, " + fi + done <<< "$caa" + all_caa=${all_caa%, } # strip trailing comma + pr_italic "$(out_row_aligned_max_width "$all_caa" "$indent " $TERM_WIDTH)" + fileout "${jsonID}${json_postfix}" "OK" "$all_caa" + else + pr_svrty_low "not offered" + fileout "${jsonID}${json_postfix}" "LOW" "--" + fi fi outln @@ -22101,6 +22115,8 @@ get_local_a() { # check_resolver_bins() { local saved_openssl_conf="$OPENSSL_CONF" + local testhost=localhost + local str="" OPENSSL_CONF="" # see https://github.com/testssl/testssl.sh/issues/134 type -p dig &> /dev/null && HAS_DIG=true @@ -22125,12 +22141,36 @@ check_resolver_bins() { HAS_DIG_NOIDNOUT=true fi fi + + # Pre-checking the following for HTTPS RR, see get_https_rrecord() + if "$HAS_DIG"; then + str=$(dig +short $testhost HTTPS) + if [[ -z "$str" ]] && [[ ! "$str" =~ 127.0.0.1 ]] ; then + HAS_DIG_HTTPS=true + fi + elif "$HAS_DRILL"; then + if drill $testhost HTTPS | grep -Eq 'IN.*HTTPS'; then + HAS_DRILL_HTTPS=true + fi + elif "$HAS_HOST"; then + host -t HTTPS $testhost 2>&1 | grep -q 'invalid type' + if [[ $? -ne 0 ]]; then + HAS_HOST_HTTPS=true + fi + elif "$HAS_NSLOOKUP"; then + nslookup -type=HTTPS $testhost | grep -q 'unknown query type' + if [[ $? -ne 0 ]]; then + HAS_NSLOOKUP_HTTPS=true + fi + fi + OPENSSL_CONF="$saved_openssl_conf" # see https://github.com/testssl/testssl.sh/issues/134 return 0 } # arg1: a host name. Returned will be 0-n IPv4 addresses # watch out: $1 can also be a cname! --> all checked +# get_a_record() { local ip4="" local saved_openssl_conf="$OPENSSL_CONF" @@ -22184,6 +22224,7 @@ get_a_record() { # arg1: a host name. Returned will be 0-n IPv6 addresses # watch out: $1 can also be a cname! --> all checked +# get_aaaa_record() { local ip6="" local saved_openssl_conf="$OPENSSL_CONF" @@ -22233,9 +22274,12 @@ get_aaaa_record() { echo "$ip6" } + # RFC6844: DNS Certification Authority Authorization (CAA) Resource Record # arg1: domain to check for -get_caa_rr_record() { +#FIXME: should be refactored, see get_https_rrecord() +# +get_caa_rrecord() { local raw_caa="" local hash len line local -i len_caa_property @@ -22263,12 +22307,16 @@ get_caa_rr_record() { raw_caa="$(drill $1 type257 | awk '/'"^${1}"'.*CAA/ { print $5,$6,$7 }')" elif "$HAS_HOST"; then raw_caa="$(host -t type257 $1)" - if grep -Ewvq "has no CAA|has no TYPE257" <<< "$raw_caa"; then - raw_caa="$(sed -e 's/^.*has CAA record //' -e 's/^.*has TYPE257 record //' <<< "$raw_caa")" + if [[ "$raw_caa" =~ "has no CAA|has no TYPE257" ]]; then + raw_caa="" + else + raw_caa="${raw_caa/$1 has CAA record /}" + raw_caa="${raw_caa/$1 has TYPE257 record /}" fi elif "$HAS_NSLOOKUP"; then raw_caa="$(strip_lf "$(nslookup -type=type257 $1 | grep -w rdata_257)")" if [[ -n "$raw_caa" ]]; then + #FIXME: modernize here or see HTTPS RR raw_caa="$(sed 's/^.*rdata_257 = //' <<< "$raw_caa")" fi else @@ -22311,11 +22359,160 @@ get_caa_rr_record() { return 1 fi -# to do: +#TODO: # 4: check whether $1 is a CNAME and take this return 0 } +# Service Binding and Parameter Specification via the DNS (SVCB and HTTPS Resource Records). +# https://www.rfc-editor.org/rfc/rfc9460.html +# arg1: domain to check for +# +get_https_rrecord() { + local raw_https="" + local hash len line + local -i len_https_property + local https_property_name + local https_property_value + local saved_openssl_conf="$OPENSSL_CONF" + local all_https="" + local noidnout="" + local svc_priority="" + + [[ -n "$NODNS" ]] && return 2 # if minimum DNS lookup was instructed, leave here + "$HAS_DIG_NOIDNOUT" && noidnout="+noidnout" + + # There's a) the possibility to query HTTPS RR records directly like "dig +short HTTPS dev.testssl.sh", + # "drill HTTPS FQDN" or "nslookup -type=HTTPS FQDN". This works for newer binaries only, unfortunately. + # On top of that b) there's also an extended format which e.g. cloudflare uses: + # $ host -t type65 testssl.net + # testssl.net has TYPE65 record \# 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041A70020002057F87361C7B5A3B8CD3C028892690D35 2863623DAD4E03D33B231A4C3C8BB02B0004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A + # $ host -t HTTPS testssl.net + # testssl.net has HTTPS record 1 . alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 ech=AEX+DQBBpwAgACBX+HNhx7WjuM08AoiSaQ01KGNiPa1OA9M7IxpMPIuwKwAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a + # ECH is the encrypted client hello --> for esni (https://datatracker.ietf.org/doc/draft-ietf-tls-esni/) + # Nice description: https://www.netmeister.org/blog/https-rrs.html + + # Thus we try first whether we can query the HTTPS records directly as this gives us that already + # in clear text and also we can avoid to parse the encoded format. We'll do that as a fallback but + # at this moment we're trying to scrape only the values alpn from it, if they come first. + + OPENSSL_CONF="" + if "$HAS_DIG_HTTPS"; then + text_httpsrr=$(dig +short +search +timeout=3 +tries=3 $noidnout HTTPS "$1" 2>/dev/null) + elif "$HAS_DRILL_HTTPS"; then + text_httpsrr=$(drill -Q HTTPS $1 2>/dev/null) + elif "$HAS_HOST_HTTPS"; then + text_httpsrr=$(host -t HTTPS $1 2>/dev/null) + text_httpsrr=${text_httpsrr#*record } + elif "$HAS_NSLOOKUP_HTTPS"; then # from 4th field onwards \/ + text_httpsrr=$(nslookup -type=HTTPS $1 | awk '/'"^${1}"'.*rdata_65// { print substr($0,index($0,$4)) }') + fi + + # Now we need to try parsing the raw output + + # If there's a type65 record there are 2x3 output formats, mostly depending on age of distribution + # -- roughly that's the difference between text and binary format -- and the type of DNS client + + # for host: + # 1) 'google.com has HTTPS record 1 . alpn="h2,h3" ' + # 2) 'google.com has TYPE65 record \# 13 0001000001000602683202683 ' + + # for drill and dig it's like + #1) google.com. 18665 IN TYPE65 \# 13 00010000010006026832026833 + #2) google.com. 18301 IN HTTPS 1 . alpn="h2,h3" + + # nslookup: + # 1) dev.testssl.sh rdata_65 = 1 . alpn="h2" + # 2) dev.testssl.sh rdata_65 = \# 10 00010000010003026832 + + # we normalize the output during the following so that's e.g. 1 . alpn="h2" + +# https://datatracker.ietf.org/doc/rfc9460/?include_text=1 + +#set -x + if [[ -n "$text_httpsrr" ]]; then + safe_echo "$text_httpsrr" + else + if "$HAS_DIG"; then + raw_https="$(dig $DIG_R +short +search +timeout=3 +tries=3 $noidnout type65 "$1" 2>/dev/null)" + # empty if there's no such record + elif "$HAS_DRILL"; then + raw_https="$(drill $1 type65 | grep -v '^;;' | awk '/'"^${1}"'.*TYPE65/ { print substr($0,index($0,$5)) }' )" # from 5th field onwards + # empty if there's no such record + elif "$HAS_HOST"; then + raw_https="$(host -t type65 $1)" + if [[ "$raw_https" =~ "has no HTTPS|has no TYPE65" ]]; then + raw_https="" + else + raw_https="${raw_https/$1 has HTTPS record /}" + raw_https="${raw_https/$1 has TYPE65 record /}" + fi + elif "$HAS_NSLOOKUP"; then + raw_https="$(strip_lf "$(nslookup -type=type65 $1 | awk '/'"^${1}"'.*rdata_65/ { print substr($0,index($0,$4)) }' )")" + # empty if there's no such record + else + return 1 + # No dig, drill, host, or nslookup --> complaint was elsewhere already + fi + OPENSSL_CONF="$saved_openssl_conf" # see https://github.com/drwetter/testssl.sh/issues/134 + +# Format probably: https://www.rfc-editor.org/rfc/rfc3597 (plus updates) + +# dig +short +search +timeout=3 +tries=3 +noidnout type65 dev.testssl.sh +# 1 . alpn="h2" port=443 ipv6hint=2a01:238:4308:a920:1000:0:b:1337 +# +# 36 000100000100030268320003000201BB000600102A0102384308A920 10000000000B1337 +# alpn| L h 2 443 2a010238... L=len + +# dig +short +search +timeout=3 +tries=3 +noidnout HTTPS testssl.net (split over a couple of lines) +# +# 1. alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 +# 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041F3002000202BD0935ED66980C1862F2570C0D6014D +# alpn| L h 3 L h 2 |IPv4#1||IPv4#2| + +# ech=AEX+DQBBzgAgACBQGA9EFbz+PkJAXSXtcqJluxLlhxIgzhJ+GhTtRd4nJQAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a +# 733A7CFAAEA5E4DD9CA43D4C24199E330004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A +# | cloudflare-ech.com | IPv6#1 #IPv6#2 + + if [[ -z "$raw_https" ]]; then + return 1 + elif [[ "$raw_https" =~ \#\ [0-9][0-9] ]]; then + while read hash len line ;do + # \# 10 00010000010003026832 + if [[ "${line:0:4}" == 0001 ]]; then # marker to proceed, belongs to SvcPriority, see rfc9460, 2.1 + svc_priority=$(printf "%0d" "$((10#${line:2:2}))") # 1 is most often, (probably not needed) type casting. 0 is alias + if [[ ${line:8:2} != 01 ]]; then # Then comes SvcParamKeys, see rfc 14.3.2 which should be alpn=-1 + continue # If the first element is not alpn, next iteration of loop will fail. + fi # Should we care as SvcParamKey!=alpn doesn't seems not very common? + + xlen_https_property=${line:12:2} # length of alpn entries + https_property_value=${line:16:4} + https_property_name=$(hex2ascii $https_property_value) + if [[ $xlen_https_property != 03 ]]; then # 06 would be another entry + https_property_value=${line:22:4} #FIXME: we can't cope with three entries yet + https_property_name="${https_property_name},$(hex2ascii $https_property_value)" + fi + echo $https_property_name + +# len_https_property=$((len_https_property*2)) # =>word! Now get name from 4th and value from 4th+len position... +# line="${line/ /}" # especially with iodefs there's a blank in the string which we just skip +# https_property_name="$(hex2ascii ${line:4:$len_https_property})" +# https_property_value="$(hex2ascii "${line:$((4+len_https_property)):100}")" + else + outln "please report unknown HTTPS RR $line with flag @ $NODE" + return 7 + fi + done <<< "$raw_https" + else + safe_echo "$raw_https" + fi + fi +#set +x + + return 0 +} + + # arg1: domain to check for. Returned will be the MX record as a string get_mx_record() { local mx="" @@ -23131,6 +23328,33 @@ determine_optimal_proto() { } +dns_https_rr () { + local jsonID="DNS_HTTPS_rrecord" + local https_rr="" + local indent="" + + out "$indent"; pr_bold " DNS HTTPS RR"; out " (experim.) " + if [[ -n "$NODNS" ]]; then + out "(instructed to minimize/skip DNS queries)" + fileout "${jsonID}" "INFO" "check skipped as instructed" + elif "$DNS_VIA_PROXY"; then + out "(instructed to use the proxy for DNS only)" + fileout "${jsonID}" "INFO" "check skipped as instructed (proxy)" + else + https_rr="$(get_https_rrecord $NODE)" + if [[ -n "$https_rr" ]]; then + pr_svrty_good "yes" ; out " " + prln_italic "$(out_row_aligned_max_width "$https_rr" "$indent " $TERM_WIDTH)" + fileout "${jsonID}" "OK" "$https_rr" + else + outln "--" + fileout "${jsonID}" "INFO" " no resource record found" + fi + fi +} + + + # Check messages which needed to be processed. I.e. those which would have destroyed the nice # screen output and thus havve been postponed. This is just an idea and is only used once # but can be extended in the future. An array might be more handy @@ -23181,7 +23405,7 @@ determine_service() { fi GET_REQ11="GET $URL_PATH HTTP/1.1\r\nHost: $NODE\r\nUser-Agent: $ua\r\n${basicauth_header}${reqheader}Accept-Encoding: identity\r\nAccept: */*\r\nConnection: Close\r\n\r\n" determine_optimal_proto - # returns always 0: + # returns always 0 and sets $SERVICE service_detection $OPTIMAL_PROTO check_msg else # STARTTLS @@ -23254,7 +23478,7 @@ determine_service() { determine_optimal_sockets_params determine_optimal_proto "$1" - out " Service set:$CORRECT_SPACES STARTTLS via " + pr_bold " Service set"; out ":$CORRECT_SPACES STARTTLS via " out "$(toupper "$protocol")" [[ "$protocol" == mysql ]] && out " (experimental)" fileout "service" "INFO" "$protocol" @@ -23268,7 +23492,6 @@ determine_service() { # It comes handy later also for STARTTLS injection to define this global. When we do banner grabbing # or replace service_detection() we might not need that anymore SERVICE=$protocol - fi tmpfile_handle ${FUNCNAME[0]}.txt @@ -23334,7 +23557,7 @@ display_rdns_etc() { outln "$PROXYIP:$PROXYPORT " fi if [[ $(count_words "$IPADDRs2SHOW") -gt 1 ]]; then - out " Further IP addresses: $CORRECT_SPACES" + pr_bold " Further IP addresses"; out ": $CORRECT_SPACES" for ip in $IPADDRs2SHOW; do if [[ "$ip" == $NODEIP ]] || [[ "[$ip]" == $NODEIP ]]; then continue @@ -23355,11 +23578,12 @@ display_rdns_etc() { outln " A record via: $CORRECT_SPACES supplied IP \"$CMDLINE_IP\"" fi fi + pr_bold " rDNS " if [[ "$rDNS" =~ instructed ]]; then - out "$(printf " %-23s " "rDNS ($nodeip):")" + out "$(printf "%-19s" "($nodeip):")" out "$rDNS" elif [[ -n "$rDNS" ]]; then - out "$(printf " %-23s " "rDNS ($nodeip):")" + out "$(printf "%-19s" "($nodeip):")" out "$(out_row_aligned_max_width "$rDNS" " $CORRECT_SPACES" $TERM_WIDTH)" fi }