From b8aadef73769ff68084a6ace8fa32ee1452ffb46 Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Tue, 8 Jul 2025 23:47:19 +0200 Subject: [PATCH 1/6] For Mac: use homebrew's openssl (3.2) ... so that we have a comparison between OpenSSL and LibreSSL. Otherwise this test would be completely futile for MacOS. Also change the displayed text. --- t/12_diff_opensslversions.t | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/t/12_diff_opensslversions.t b/t/12_diff_opensslversions.t index 5262463..91d3262 100755 --- a/t/12_diff_opensslversions.t +++ b/t/12_diff_opensslversions.t @@ -20,6 +20,8 @@ my $uri="google.com"; my $diff=""; my $distro_openssl="/usr/bin/openssl"; my @args=""; +# that can be done better but I am a perl n00b ;-) +my $os=`perl -e 'print "$^O";'`; die "Unable to open $prg" unless -f $prg; die "Unable to open $distro_openssl" unless -f $distro_openssl; @@ -29,13 +31,22 @@ unlink $csvfile; unlink $csvfile2; #1 run -printf "\n%s\n", "Diff test IPv4 with supplied openssl against \"$uri\""; -@args="$prg $check2run $csvfile $uri >/dev/null"; + +if ( $os eq "linux" ){ + printf "\n%s\n", "Test with supplied openssl against \"$uri\" and save it"; + @args="$prg $check2run $csvfile $uri >/dev/null"; +} elsif ( $os eq "darwin" ){ + # macos silicon doesn't have ~/bin/openssl.Darwin.arm64 binary so we use the + # homebrew version which was moved to /opt/homebrew/bin/openssl.NOPE in + # .github/workflows/unit_tests_macos.yml + printf "\n%s\n", "Test with homebrew's openssl 3.5.x against \"$uri\" and save it"; + @args="$prg $check2run $csvfile --openssl /opt/homebrew/bin/openssl.NOPE $uri >/dev/null"; +} system("@args") == 0 or die ("FAILED: \"@args\""); # 2 -printf "\n%s\n", "Diff test IPv4 with $distro_openssl against \"$uri\""; +printf "\n%s\n", "Test with $distro_openssl against \"$uri\" and save it"; @args="$prg $check2run $csvfile2 --openssl=$distro_openssl $uri >/dev/null"; system("@args") == 0 or die ("FAILED: \"@args\" "); From 4f52cbbedf78413ec4c8ae03e54cea66a274df49 Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Wed, 9 Jul 2025 13:31:18 +0200 Subject: [PATCH 2/6] Copy LibreSSL change of naming scheme from t/61_diff_testsslsh.t ... also if NOW only ciphers seem affected. --- t/12_diff_opensslversions.t | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/t/12_diff_opensslversions.t b/t/12_diff_opensslversions.t index 91d3262..ed5b8ce 100755 --- a/t/12_diff_opensslversions.t +++ b/t/12_diff_opensslversions.t @@ -78,6 +78,21 @@ $cat_csvfile2 =~ s/.nonce-.* //g; $cat_csvfile =~ s/","google.com\/.*","443/","google.com","443/g; $cat_csvfile2 =~ s/","google.com\/.*","443/","google.com","443/g; +# Now address the differences for LibreSSL, see t/61_diff_testsslsh.t +# +# MacOS / LibreSSL has different OpenSSL names for TLS 1.3 ciphers. That should be rather solved in +# testssl.sh, see #2763. But for now we do this here. +$cat_csvfile =~ s/AEAD-AES128-GCM-SHA256/TLS_AES_128_GCM_SHA256/g; +$cat_csvfile =~ s/AEAD-AES256-GCM-SHA384/TLS_AES_256_GCM_SHA384/g +# this is a bit ugly but otherwise the line cipher-tls1_3_x1303 with the CHACHA20 cipher misses a space +$cat_csvfile =~ s/x1303 AEAD-CHACHA20-POLY1305-SHA256/x1303 TLS_CHACHA20_POLY1305_SHA256 /g; +# now the other lines, where we don't need to insert the additional space: +$cat_csvfile =~ s/AEAD-CHACHA20-POLY1305-SHA256/TLS_CHACHA20_POLY1305_SHA256/g; + +# Same with ECDH bit length +$cat_csv =~ s/ECDH 253/ECDH 256/g; +$baseline_csv =~ s/ECDH 253/ECDH 256/g; + $diff = diff \$cat_csvfile, \$cat_csvfile2; # Compare the differences -- and print them if there were any From 822c6b31d7b321f981c0a800d2016c68b6216363 Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Wed, 9 Jul 2025 15:22:48 +0200 Subject: [PATCH 3/6] fix stnax errors --- t/12_diff_opensslversions.t | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/t/12_diff_opensslversions.t b/t/12_diff_opensslversions.t index ed5b8ce..b144ac5 100755 --- a/t/12_diff_opensslversions.t +++ b/t/12_diff_opensslversions.t @@ -83,15 +83,13 @@ $cat_csvfile2 =~ s/","google.com\/.*","443/","google.com","443/g; # MacOS / LibreSSL has different OpenSSL names for TLS 1.3 ciphers. That should be rather solved in # testssl.sh, see #2763. But for now we do this here. $cat_csvfile =~ s/AEAD-AES128-GCM-SHA256/TLS_AES_128_GCM_SHA256/g; -$cat_csvfile =~ s/AEAD-AES256-GCM-SHA384/TLS_AES_256_GCM_SHA384/g +$cat_csvfile =~ s/AEAD-AES256-GCM-SHA384/TLS_AES_256_GCM_SHA384/g; # this is a bit ugly but otherwise the line cipher-tls1_3_x1303 with the CHACHA20 cipher misses a space $cat_csvfile =~ s/x1303 AEAD-CHACHA20-POLY1305-SHA256/x1303 TLS_CHACHA20_POLY1305_SHA256 /g; # now the other lines, where we don't need to insert the additional space: $cat_csvfile =~ s/AEAD-CHACHA20-POLY1305-SHA256/TLS_CHACHA20_POLY1305_SHA256/g; - # Same with ECDH bit length -$cat_csv =~ s/ECDH 253/ECDH 256/g; -$baseline_csv =~ s/ECDH 253/ECDH 256/g; +$cat_csvfile =~ s/ECDH 253/ECDH 256/g; $diff = diff \$cat_csvfile, \$cat_csvfile2; From 1f589565c7746555724e749925793a78f949aa4b Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Wed, 9 Jul 2025 16:17:28 +0200 Subject: [PATCH 4/6] we changed above the ECDH bit length already --- t/12_diff_opensslversions.t | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/t/12_diff_opensslversions.t b/t/12_diff_opensslversions.t index b144ac5..7163d08 100755 --- a/t/12_diff_opensslversions.t +++ b/t/12_diff_opensslversions.t @@ -88,8 +88,7 @@ $cat_csvfile =~ s/AEAD-AES256-GCM-SHA384/TLS_AES_256_GCM_SHA384/g; $cat_csvfile =~ s/x1303 AEAD-CHACHA20-POLY1305-SHA256/x1303 TLS_CHACHA20_POLY1305_SHA256 /g; # now the other lines, where we don't need to insert the additional space: $cat_csvfile =~ s/AEAD-CHACHA20-POLY1305-SHA256/TLS_CHACHA20_POLY1305_SHA256/g; -# Same with ECDH bit length -$cat_csvfile =~ s/ECDH 253/ECDH 256/g; +# we changed above the ECDH bit length already $diff = diff \$cat_csvfile, \$cat_csvfile2; From 815e737bfe12479340e783e0e3d3d087c0b4841e Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Wed, 9 Jul 2025 18:10:30 +0200 Subject: [PATCH 5/6] Check the correct file --- t/12_diff_opensslversions.t | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/t/12_diff_opensslversions.t b/t/12_diff_opensslversions.t index 7163d08..8b0a652 100755 --- a/t/12_diff_opensslversions.t +++ b/t/12_diff_opensslversions.t @@ -45,7 +45,7 @@ if ( $os eq "linux" ){ system("@args") == 0 or die ("FAILED: \"@args\""); -# 2 +# 2 (LibreSSL in case of MacOS) printf "\n%s\n", "Test with $distro_openssl against \"$uri\" and save it"; @args="$prg $check2run $csvfile2 --openssl=$distro_openssl $uri >/dev/null"; system("@args") == 0 @@ -78,17 +78,20 @@ $cat_csvfile2 =~ s/.nonce-.* //g; $cat_csvfile =~ s/","google.com\/.*","443/","google.com","443/g; $cat_csvfile2 =~ s/","google.com\/.*","443/","google.com","443/g; -# Now address the differences for LibreSSL, see t/61_diff_testsslsh.t -# -# MacOS / LibreSSL has different OpenSSL names for TLS 1.3 ciphers. That should be rather solved in -# testssl.sh, see #2763. But for now we do this here. -$cat_csvfile =~ s/AEAD-AES128-GCM-SHA256/TLS_AES_128_GCM_SHA256/g; -$cat_csvfile =~ s/AEAD-AES256-GCM-SHA384/TLS_AES_256_GCM_SHA384/g; -# this is a bit ugly but otherwise the line cipher-tls1_3_x1303 with the CHACHA20 cipher misses a space -$cat_csvfile =~ s/x1303 AEAD-CHACHA20-POLY1305-SHA256/x1303 TLS_CHACHA20_POLY1305_SHA256 /g; -# now the other lines, where we don't need to insert the additional space: -$cat_csvfile =~ s/AEAD-CHACHA20-POLY1305-SHA256/TLS_CHACHA20_POLY1305_SHA256/g; -# we changed above the ECDH bit length already + +if ( $os eq "darwin" ){ + # Now address the differences for LibreSSL, see t/61_diff_testsslsh.t + # + # MacOS / LibreSSL has different OpenSSL names for TLS 1.3 ciphers. That should be rather solved in + # testssl.sh, see #2763. But for now we do this here. + $cat_csvfile2 =~ s/AEAD-AES128-GCM-SHA256/TLS_AES_128_GCM_SHA256/g; + $cat_csvfile2 =~ s/AEAD-AES256-GCM-SHA384/TLS_AES_256_GCM_SHA384/g; + # this is a bit ugly but otherwise the line cipher-tls1_3_x1303 with the CHACHA20 cipher misses a space + $cat_csvfile2 =~ s/x1303 AEAD-CHACHA20-POLY1305-SHA256/x1303 TLS_CHACHA20_POLY1305_SHA256 /g; + # now the other lines, where we don't need to insert the additional space: + $cat_csvfile2 =~ s/AEAD-CHACHA20-POLY1305-SHA256/TLS_CHACHA20_POLY1305_SHA256/g; + # we changed above the ECDH bit length already +} $diff = diff \$cat_csvfile, \$cat_csvfile2; From d63617c0f6658d14369ef5d8c411847838da7c5d Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Thu, 10 Jul 2025 09:14:08 +0200 Subject: [PATCH 6/6] make it compatible when run locally --- t/12_diff_opensslversions.t | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/t/12_diff_opensslversions.t b/t/12_diff_opensslversions.t index 8b0a652..e866df1 100755 --- a/t/12_diff_opensslversions.t +++ b/t/12_diff_opensslversions.t @@ -31,21 +31,34 @@ unlink $csvfile; unlink $csvfile2; #1 run - if ( $os eq "linux" ){ + # Comparison ~/bin/openssl.Linux.x86_64 printf "\n%s\n", "Test with supplied openssl against \"$uri\" and save it"; @args="$prg $check2run $csvfile $uri >/dev/null"; } elsif ( $os eq "darwin" ){ - # macos silicon doesn't have ~/bin/openssl.Darwin.arm64 binary so we use the + # MacOS silicon doesn't have ~/bin/openssl.Darwin.arm64 binary so we use the # homebrew version which was moved to /opt/homebrew/bin/openssl.NOPE in - # .github/workflows/unit_tests_macos.yml - printf "\n%s\n", "Test with homebrew's openssl 3.5.x against \"$uri\" and save it"; - @args="$prg $check2run $csvfile --openssl /opt/homebrew/bin/openssl.NOPE $uri >/dev/null"; + # .github/workflows/unit_tests_macos.yml . This gives us instead a comparison + # check from OpenSSL + # If this will be run outside GH actions, i.e. locally, we provide a fallback to + # /opt/homebrew/bin/openssl or just leave this thing + if ( -x "/opt/homebrew/bin/openssl.NOPE" ) { + printf "\n%s\n", "Test with homebrew's openssl 3.5.x against \"$uri\" and save it"; + @args="$prg $check2run $csvfile --openssl /opt/homebrew/bin/openssl.NOPE $uri >/dev/null"; + } + elsif ( -x "/opt/homebrew/bin/openssl" ) { + printf "\n%s\n", "Test with homebrew's openssl 3.5.x against \"$uri\" and save it"; + @args="$prg $check2run $csvfile --openssl /opt/homebrew/bin/openssl $uri >/dev/null"; + } + else { + die ("No alternative version to LibreSSL found"); + } } system("@args") == 0 or die ("FAILED: \"@args\""); -# 2 (LibreSSL in case of MacOS) + +# 2 (LibreSSL in case of MacOS, /usr/bin/openssl for Linux) printf "\n%s\n", "Test with $distro_openssl against \"$uri\" and save it"; @args="$prg $check2run $csvfile2 --openssl=$distro_openssl $uri >/dev/null"; system("@args") == 0