Restructure IDN support, DNS improvements

In order to not repeatedly call check_resolver_bins() the function
was moved to top level. As each check in check_resolver_bins now
is only executed once, it should also work faster. Each get_*_record()
now uses HAS_ variables only.

Also check_resolver_bins() contain now the check whether
idn/idn2 support is available.

Then the IDN URI conversion snipplet was moved to the final function
parse_hn_port() which does operations in the URI supplied.

testssl.sh

@@ -355,6 +355,15 @@ HAS_CHACHA20=false
 HAS_AES128_GCM=false
 HAS_AES256_GCM=false
 HAS_ZLIB=false
+HAS_DIG=false
+HAS_HOST=false
+HAS_DRILL=false
+HAS_NSLOOKUP=false
+HAS_IDN=false
+HAS_IDN2=false
+HAS_AVAHIRESOLVE=false
+HAS_DIG_NOIDNOUT=false
+
  OSSL_CIPHERS_S=""
 PORT=443                                # unless otherwise auto-determined, see below
 NODE=""
@@ -16981,6 +16990,15 @@ HAS_LMTP: $HAS_LMTP
 HAS_NNTP: $HAS_NNTP
 HAS_IRC: $HAS_IRC
 
+HAS_DIG: $HAS_DIG
+HAS_HOST: $HAS_HOST
+HAS_DRILL: $HAS_DRILL
+HAS_NSLOOKUP: $HAS_NSLOOKUP
+HAS_IDN: $HAS_IDN
+HAS_IDN2: $HAS_IDN2
+HAS_AVAHIRESOLVE: $HAS_AVAHIRESOLVE
+HAS_DIG_NOIDNOUT: $HAS_DIG_NOIDNOUT
+
 PATH: $PATH
 PROG_NAME: $PROG_NAME
 TESTSSL_INSTALL_DIR: $TESTSSL_INSTALL_DIR
@@ -17286,13 +17304,12 @@ ignore_no_or_lame() {
 # arg1: URI
 parse_hn_port() {
      local tmp_port
+     local node_tmp=""
 
      NODE="$1"
      NODE="${NODE/https\:\/\//}"        # strip "https"
      NODE="${NODE%%/*}"                 # strip trailing urlpath
      NODE="${NODE%%.}"                  # strip trailing "." if supplied
-
-     # if there's a trailing ':' probably a starttls/application protocol was specified
      if grep -q ':$' <<< "$NODE"; then
           if grep -wq http <<< "$NODE"; then
                fatal "\"http\" is not what you meant probably" $ERR_CMDLINE
@@ -17300,8 +17317,7 @@ parse_hn_port() {
                fatal "\"$1\" is not a valid URI" $ERR_CMDLINE
           fi
      fi
-
+     # Was an IPv6 address supplied like [AA:BB:CC::]:port ?
-     # was the address supplied like [AA:BB:CC::]:port ?
      if grep -q ']' <<< "$NODE"; then
           tmp_port=$(printf "$NODE" | sed 's/\[.*\]//' | sed 's/://')
           # determine v6 port, supposed it was supplied additionally
@@ -17315,9 +17331,34 @@ parse_hn_port() {
           grep -q ':' <<< "$NODE" && \
                PORT=$(sed 's/^.*\://' <<< "$NODE") && NODE=$(sed 's/\:.*$//' <<< "$NODE")
      fi
+
+     # We check for non-ASCII chars now. If there are some we'll try to convert it if IDN/IDN2 is installed
+     # If not, we'll continue. Hoping later that dig can use it. If not the error handler will tell
+     # Honestly we don't care whether it's IDN2008 or IDN2003 or Emoji domains as long as it works.
+     # So we try to resolve anything supplied. If it can't  our resolver error handler takes care
+     if [[ "$NODE" == *[![:ascii:]]* ]]; then
+          if ! "$HAS_IDN2" && ! "$HAS_IDN"; then
+               prln_warning " URI contains non-ASCII characters and libidn/libidn2 not available."
+               outln " Trying to feed the resolver without converted \"$NODE\" ...\n"
+               #ToDo: fileout is missing
+               node_tmp="$NODE"
+          elif "$HAS_IDN2"; then
+               node_tmp="$(idn2 "$NODE" 2>/dev/null)"
+          fi
+          if "$HAS_IDN" && [[ -z "$node_tmp" ]]; then
+               node_tmp="$(idn "$NODE" 2>/dev/null)"
+          fi
+          if [[ -z "$node_tmp" ]]; then
+               prln_warning " URI contains non-ASCII characters and IDN conversion failed."
+               outln " Trying to feed the resolver without converted \"$NODE\" ...\n"
+               #ToDo: fileout is missing
+               node_tmp="$NODE"
+          fi
+          NODE="$node_tmp"
+     fi
+
      debugme echo $NODE:$PORT
      SNI="-servername $NODE"
-
      URL_PATH=$(sed 's/https:\/\///' <<< "$1" | sed 's/'"${NODE}"'//' | sed 's/.*'"${PORT}"'//')      # remove protocol and node part and port
      URL_PATH=$(sed 's/\/\//\//g' <<< "$URL_PATH")          # we rather want // -> /
      [[ -z "$URL_PATH" ]] && URL_PATH="/"
@@ -17384,11 +17425,26 @@ get_local_a() {
      fi
 }
 
-# does a hard exit if no lookup binary is provided
+# Does a hard exit if no lookup binary is provided
+# Checks for IDN capabilities also
+#
 check_resolver_bins() {
-     if ! type -p dig &> /dev/null && ! type -p host &> /dev/null && ! type -p drill &> /dev/null && ! type -p nslookup &>/dev/null; then
+     type -p dig   &> /dev/null &&  HAS_DIG=true
+     type -p dig   &> /dev/null &&  HAS_HOST=true
+     type -p drill &> /dev/null &&  HAS_DRILL=true
+     type -p nslookup &> /dev/null && HAS_NSLOOKUP=true
+     type -p avahi-resolve &>/dev/null && HAS_AVAHIRESOLVE=true
+     type -p idn  &>/dev/null && HAS_IDN=true
+     type -p idn2 &>/dev/null && HAS_IDN2=true
+
+     if ! "$HAS_DIG" && ! "$HAS_HOST" && "$HAS_DRILL" && ! "$HAS_NSLOOKUP"; then
           fatal "Neither \"dig\", \"host\", \"drill\" or \"nslookup\" is present" $ERR_DNSBIN
      fi
+     if "$HAS_DIG"; then
+          if dig +noidnout -t a 2>&1 | grep -Eqv 'Invalid option: \+noidnout|IDN support not enabled'; then
+               HAS_DIG_NOIDNOUT=true
+          fi
+     fi
      return 0
 }
 
@@ -17410,34 +17466,27 @@ get_a_record() {
           return 0
      fi
      OPENSSL_CONF=""                         # see https://github.com/drwetter/testssl.sh/issues/134
-     check_resolver_bins
      if [[ "$NODE" == *.local ]]; then
-          if type -p avahi-resolve &>/dev/null; then
+          if "$HAS_AVAHIRESOLVE"; then
                ip4=$(filter_ip4_address $(avahi-resolve -4 -n "$1" 2>/dev/null | awk '{ print $2 }'))
-          elif type -p dig &>/dev/null; then
+          elif "$HAS_DIG"; then
                ip4=$(filter_ip4_address $(dig @224.0.0.251 -p 5353 +short -t a +notcp "$1" 2>/dev/null | sed '/^;;/d'))
           else
                fatal "Local hostname given but no 'avahi-resolve' or 'dig' available." $ERR_DNSBIN
           fi
      fi
-     if [[ -z "$ip4" ]]; then
+     if [[ -z "$ip4" ]] && "$HAS_DIG"; then
-          if type -p dig &> /dev/null ; then
           ip4=$(filter_ip4_address $(dig +timeout=2 +tries=2 +short -t a "$1" 2>/dev/null | awk '/^[0-9]/ { print $1 }'))
      fi
-     fi
+     if [[ -z "$ip4" ]] && "$HAS_HOST"; then
-     if [[ -z "$ip4" ]]; then
-          type -p host &> /dev/null && \
           ip4=$(filter_ip4_address $(host -t a "$1" 2>/dev/null | awk '/address/ { print $NF }'))
      fi
-     if [[ -z "$ip4" ]]; then
+     if [[ -z "$ip4" ]] && "$HAS_DRILL"; then
-          type -p drill &> /dev/null && \
           ip4=$(filter_ip4_address $(drill a "$1" | awk '/ANSWER SECTION/,/AUTHORITY SECTION/ { print $NF }' | awk '/^[0-9]/'))
      fi
-     if [[ -z "$ip4" ]]; then
+     if [[ -z "$ip4" ]] && "$HAS_NSLOOKUP"; then
-          if type -p nslookup &>/dev/null; then
           ip4=$(filter_ip4_address $(strip_lf "$(nslookup -querytype=a "$1" 2>/dev/null | awk '/^Name/ { getline; print $NF }')"))
      fi
-     fi
      OPENSSL_CONF="$saved_openssl_conf"      # see https://github.com/drwetter/testssl.sh/issues/134
      echo "$ip4"
 }
@@ -17458,23 +17507,22 @@ get_aaaa_record() {
           # we need also this here as get_aaaa_record is always called after get_a_record and we want to handle this at a low level
           return 0
      fi
-     check_resolver_bins
      if [[ -z "$ip6" ]]; then
           if [[ "$NODE" == *.local ]]; then
-               if type -p avahi-resolve &>/dev/null; then
+               if "$HAS_AVAHIRESOLVE"; then
                     ip6=$(filter_ip6_address $(avahi-resolve -6 -n "$1" 2>/dev/null | awk '{ print $2 }'))
-               elif type -p dig &>/dev/null; then
+               elif "$HAS_DIG"; then
                     ip6=$(filter_ip6_address $(dig @ff02::fb -p 5353 -t aaaa +short +notcp "$NODE"))
                else
                     fatal "Local hostname given but no 'avahi-resolve' or 'dig' available." $ERR_DNSBIN
                fi
-          elif type -p dig &> /dev/null; then
+          elif "$HAS_DIG"; then
                ip6=$(filter_ip6_address $(dig +short +timeout=2 +tries=2 -t aaaa "$1" 2>/dev/null | awk '/^[0-9]/ { print $1 }'))
-          elif type -p host &> /dev/null ; then
+          elif "$HAS_HOST"; then
                ip6=$(filter_ip6_address $(host -t aaaa "$1" | awk '/address/ { print $NF }'))
-          elif type -p drill &> /dev/null; then
+          elif "$HAS_DRILL"; then
                ip6=$(filter_ip6_address $(drill aaaa "$1" | awk '/ANSWER SECTION/,/AUTHORITY SECTION/ { print $NF }' | awk '/^[0-9]/'))
-          elif type -p nslookup &>/dev/null; then
+          elif "$HAS_NSLOOKUP"; then
                ip6=$(filter_ip6_address $(strip_lf "$(nslookup -type=aaaa "$1" 2>/dev/null | awk '/'"^${a}"'.*AAAA/ { print $NF }')"))
           fi
      fi
@@ -17500,18 +17548,17 @@ get_caa_rr_record() {
      # for dig +short the output always starts with '0 issue [..]' or '\# 19 [..]' so we normalize thereto to keep caa_flag, caa_property
      # caa_property then has key/value pairs, see https://tools.ietf.org/html/rfc6844#section-3
      OPENSSL_CONF=""
-     check_resolver_bins
+     if "$HAS_DIG"; then
-     if type -p dig &> /dev/null; then
           raw_caa="$(dig +timeout=3 +tries=3 $1 type257 +short | awk '{ print $1" "$2" "$3 }')"
           # empty if no CAA record
-     elif type -p drill &> /dev/null; then
+     elif "$HAS_DRILL"; then
           raw_caa="$(drill $1 type257 | awk '/'"^${1}"'.*CAA/ { print $5,$6,$7 }')"
-     elif type -p host &> /dev/null; then
+     elif "$HAS_HOST"; then
           raw_caa="$(host -t type257 $1)"
           if grep -Ewvq "has no CAA|has no TYPE257" <<< "$raw_caa"; then
                raw_caa="$(sed -e 's/^.*has CAA record //' -e 's/^.*has TYPE257 record //' <<< "$raw_caa")"
           fi
-     elif type -p nslookup &> /dev/null; then
+     elif "$HAS_NSLOOKUP"; then
           raw_caa="$(strip_lf "$(nslookup -type=type257 $1 | grep -w rdata_257)")"
           if [[ -n "$raw_caa" ]]; then
                raw_caa="$(sed 's/^.*rdata_257 = //' <<< "$raw_caa")"
@@ -17567,17 +17614,17 @@ get_mx_record() {
      local saved_openssl_conf="$OPENSSL_CONF"
 
      OPENSSL_CONF=""                         # see https://github.com/drwetter/testssl.sh/issues/134
-     check_resolver_bins
      # we need the last two columns here
-     if type -p host &> /dev/null; then
+     if "$HAS_HOST"; then
           mxs="$(host -t MX "$1" 2>/dev/null | awk '/is handled by/ { print $(NF-1), $NF }')"
-     elif type -p dig &> /dev/null; then
+     elif "$HAS_DIG"; then
           mxs="$(dig +short -t MX "$1" 2>/dev/null | awk '/^[0-9]/ { print $1" "$2 }')"
-     elif type -p drill &> /dev/null; then
+     elif "$HAS_DRILL"; then
           mxs="$(drill mx $1 | awk '/IN[ \t]MX[ \t]+/ { print $(NF-1), $NF }')"
-     elif type -p nslookup &> /dev/null; then
+     elif "$HAS_NSLOOKUP"; then
           mxs="$(strip_lf "$(nslookup -type=MX "$1" 2>/dev/null | awk '/mail exchanger/ { print $(NF-1), $NF }')")"
      else
+          # shouldn't reach this, as we checked in the top
           fatal "No dig, host, drill or nslookup" $ERR_DNSBIN
      fi
      OPENSSL_CONF="$saved_openssl_conf"
@@ -17666,21 +17713,20 @@ determine_rdns() {
      [[ -n "$NODNS" ]] && rDNS="(instructed to minimize DNS queries)" && return 0   # PTR records were not asked for
      local nodeip="$(tr -d '[]' <<< $NODEIP)"     # for DNS we do not need the square brackets of IPv6 addresses
      OPENSSL_CONF=""                              # see https://github.com/drwetter/testssl.sh/issues/134
-     check_resolver_bins
      if [[ "$NODE" == *.local ]]; then
-          if type -p avahi-resolve &>/dev/null; then
+          if "$HAS_AVAHIRESOLVE"; then
                rDNS=$(avahi-resolve -a $nodeip 2>/dev/null | awk '{ print $2 }')
-          elif type -p dig &>/dev/null; then
+          elif "$HAS_DIG"; then
                rDNS=$(dig -x $nodeip @224.0.0.251 -p 5353 +notcp +noall +answer +short | awk '{ print $1 }')
           fi
-     elif type -p dig &> /dev/null; then
+     elif "$HAS_DIG"; then
           # 1+2 should suffice. It's a compromise for if e.g. network is down but we have a docker/localhost server
           rDNS=$(dig -x $nodeip +timeout=1 +tries=2 +noall +answer +short | awk '{ print $1 }')    # +short returns also CNAME, e.g. openssl.org
-     elif type -p host &> /dev/null; then
+     elif "$HAS_HOST"; then
           rDNS=$(host -t PTR $nodeip 2>/dev/null | awk '/pointer/ { print $NF }')
-     elif type -p drill &> /dev/null; then
+     elif "$HAS_DRILL"; then
           rDNS=$(drill -x ptr $nodeip 2>/dev/null | awk '/ANSWER SECTION/ { getline; print $NF }')
-     elif type -p nslookup &> /dev/null; then
+     elif "$HAS_NSLOOKUP"; then
           rDNS=$(strip_lf "$(nslookup -type=PTR $nodeip 2>/dev/null | grep -v 'canonical name =' | grep 'name = ' | awk '{ print $NF }' | sed 's/\.$//')")
      fi
      OPENSSL_CONF="$saved_openssl_conf"      # see https://github.com/drwetter/testssl.sh/issues/134
@@ -19365,37 +19411,8 @@ parse_cmd_line() {
      if [[ -z "$1" ]] && [[ -z "$FNAME" ]] && ! "$do_display_only"; then
           fatal "URI missing" $ERR_CMDLINE
      else
-     # What is left here is the URI. We check for non-ASCII chars first:
+     # What is left here should be the URI.
-          if [[ "$1" == *[![:ascii:]]* ]]; then
-               HAS_IDN=false
-               HAS_IDN2=false
-               HAS_NOIDNOUT=false
-               #PoC for maybe later use
-               if type -p dig &>/dev/null; then
-                    if dig +noidnout -t a 2>&1 | grep -Eqv 'Invalid option: \+noidnout|IDN support not enabled'; then
-                         HAS_NOIDNOUT=true
-                    fi
-               fi
-               type -p idn &>/dev/null && HAS_IDN=true
-               type -p idn2 &>/dev/null && HAS_IDN2=true
-               #ToDo: the user needs to know whether installing libidn(2) could help him here
-               if "$HAS_IDN2"; then
-                    URI="$(idn2 "$1" 2>/dev/null)"
-               fi
-               if "$HAS_IDN" && [[ -z "$URI" ]]; then
-                    URI="$(idn "$1" 2>/dev/null)"
-               fi
-               if [[ -z "$URI" ]]; then
-                    # fatal "URI contains non-ASCII characters, and IDN not available."
-                    pr_warning "URI contains non-ASCII characters, and IDN not available or conversion failed."
-                    outln " Trying to continue with not converted URI"
-                    #ToDo: fileout is missing
           URI="$1"
-               fi
-          else
-              URI="$1"
-          fi
-          # parameter after URI supplied:
           [[ -n "$2" ]] && fatal "URI comes last" $ERR_CMDLINE
      fi
      [[ $CMDLINE_IP == one ]] && [[ "$NODNS" == none ]] && fatal "\"--ip=one\" and \"--nodns=none\" don't work together" $ERR_CMDLINE
@@ -19627,6 +19644,7 @@ lets_roll() {
      check_proxy
      check4openssl_oldfarts
      check_bsd_mount
+     check_resolver_bins
 
      if "$do_display_only"; then
           prettyprint_local "$PATTERN2SHOW"