Merge pull request #2761 from testssl/mac_runner

PoC github action runner macos-14

.github/workflows/unit_tests_macos.yml

@@ -0,0 +1,54 @@
+---
+name: CI run MacOS
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'utils/**'
+      - 'doc/**'
+      - 'bin/**'
+      - '**.md'
+      - '**.pem'
+      - '**.pdf'
+      - '**.html'
+      - 'LICENSE'
+      - 'Dockerfile'
+      - 'Dockerfile.alpine'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: macos-14
+
+    name: PoC unit test on MacOS
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Set up system (perl, curl is installed already)
+        run: |
+          brew install socat jsonlint cpanminus
+          printf "%s\n" "----------"
+          mv -v /opt/homebrew/bin/openssl /opt/homebrew/bin/openssl.NOPE
+          printf "%s\n" "----------"
+          perl -V
+          printf "%s\n" "----------"
+          curl --version
+          printf "%s\n" "----------"
+          openssl version -a
+          printf "%s\n" "----------"
+          bash --version
+          printf "%s\n" "----------"
+
+      - name: Install perl modules
+        run: |
+          cpanm --notest Test::More
+          cpanm --notest Data::Dumper
+          cpanm --notest JSON
+          cpanm --notest Text::Diff
+
+      - name: run it
+        run: |
+          prove -v t

.github/workflows/unit_tests_ubuntu.yml

@@ -1,5 +1,5 @@
 ---
-name: testssl.sh CI with Ubuntu
+name: CI run Ubuntu
 
 on:
   pull_request:

Readme.md

@@ -2,12 +2,13 @@
 ## Intro
 ![GitHub Tag](https://img.shields.io/github/v/tag/testssl/testssl.sh)
 ![Static Badge](https://img.shields.io/badge/%2Fbin%2Fbash_-blue)
-![Static Badge](https://img.shields.io/badge/OpenSSL_-blue)
+![Static Badge](https://img.shields.io/badge/Libre+OpenSSL_-blue)
 [![License](https://img.shields.io/github/license/testssl/testssl.sh)](https://github.com/testssl/testssl.sh/LICENSE)
 ![GitHub Created At](https://img.shields.io/github/created-at/testssl/testssl.sh)
 ![GitHub last commit](https://img.shields.io/github/last-commit/testssl/testssl.sh)
 ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/testssl/testssl.sh)
-[![CI tests](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_ubuntu.yml/badge.svg)](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_ubuntu.yml)
+[![CI test Ubuntu](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_ubuntu.yml/badge.svg)](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_ubuntu.yml)
+[![CI test MacOS](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_macos.yml/badge.svg)](https://github.com/testssl/testssl.sh/actions/workflows/unit_tests_macos.yml)
 [![Docker](https://img.shields.io/docker/pulls/drwetter/testssl.sh)](https://github.com/testssl/testssl.sh/blob/3.2/Dockerfile.md)
 ![Mastodon Follow](https://img.shields.io/mastodon/follow/109319848143024146?domain=infosec.exchange)
 [![Bluesky](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fpublic.api.bsky.app%2Fxrpc%2Fapp.bsky.actor.getProfile%2F%3Factor%3Dtestssl.bsky.social&query=%24.followersCount&style=social&logo=bluesky&label=Follow%20%40testssl.sh)
@@ -30,6 +31,7 @@ cryptographic flaws.
 * Privacy: It's only you who sees the result, not a third party.
 * Freedom: It's 100% open source. You can look at the code, see what's going on.
 * The development is free and open @ GitHub and participation is welcome.
+* Unit tests ensure maturity (output is consistent, JSON is valid, runs under Linux+MacOS etc)
 
 ### License
 
@@ -52,7 +54,7 @@ but you can also use any LibreSSL or OpenSSL version.
 (silent) check for binaries is done when you start testssl.sh . System V needs probably
 to have GNU grep installed. Windows (using MSYS2, Cygwin or WSL) work too.
 
-Update notification here or @ [mastodon](https://infosec.exchange/@testssl) or [bluesky](https://bsky.app/profile/testssl.bsky.social). [twitter](https://twitter.com/drwetter) is not being used anymore.
+Update notification can be found here or most important ones @ [mastodon](https://infosec.exchange/@testssl) or [bluesky](https://bsky.app/profile/testssl.bsky.social). [twitter](https://twitter.com/drwetter) is not being used anymore.
 
 ### Installation
 

t/00_testssl_help.t

@@ -24,33 +24,42 @@ printf "\n%s\n", "Testing whether just calling \"./testssl.sh\" produces no erro
 my $info    = stat($prg);
 my $retMode = $info->mode;
 
+#1
 is($retMode & 0400, 0400, "Checking \"./testssl.sh\" for read permission");
 $tests++;
 
+#2
 is($retMode & 0100, 0100, "Checking \"./testssl.sh\" for execute permission");
 $tests++;
 
-$fileout = `timeout 10 bash $prg 2>&1`;
+$fileout = `bash $prg 2>&1`;
 my $retval=$?;
 
+#3
 unlike($fileout, qr/$error_regexp1/, "regex 1");
 $tests++;
 
+#4
 unlike($fileout, qr/$error_regexp2/, "regex 2");
 $tests++;
 
+#5
 unlike($fileout, qr/$error_regexp3/, "regex 3");
 $tests++;
 
+#6
 unlike($fileout, qr/$error_regexp4/, "regex 4");
 $tests++;
 
+#7
 unlike($fileout, qr/$error_regexp5/, "regex 5");
 $tests++;
 
+#8
 is($retval, 0, "return value should be equal zero: \"$retval\"");
 $tests++;
 
+#9
 $out=`grep -E "$debug_regexp" $prg`;
 unlike($out, qr/$debug_regexp/, "Debug RegEx");
 $tests++;

t/01_testssl_banner.t

@@ -17,27 +17,34 @@ my $error_regexp5='(syntax error|unexpected token)';
 my $good_regexp='free software([\s\S]*)USAGE w/o ANY WARRANTY([\s\S]*)OWN RISK([\s\S]*)Using([\s\S]*)ciphers([\s\S]*)built([\s\S]*)platform';
 
 printf "\n%s\n", "Testing whether just calling \"./testssl.sh --banner\" produces no error ...";
-$fileout = `timeout 10 bash ./testssl.sh --banner 2>&1`;
+$fileout = `bash ./testssl.sh --banner 2>&1`;
 my $retval=$?;
 
+#1
 unlike($fileout, qr/$error_regexp1/, "regex 1");
 $tests++;
 
+#2
 unlike($fileout, qr/$error_regexp2/, "regex 2");
 $tests++;
 
+#3
 unlike($fileout, qr/$error_regexp3/, "regex 3");
 $tests++;
 
+#4
 unlike($fileout, qr/$error_regexp4/, "regex 4");
 $tests++;
 
+#5
 unlike($fileout, qr/$error_regexp5/, "regex 5");
 $tests++;
 
+#6
 like($fileout, qr/$good_regexp/, "regex positive");
 $tests++;
 
+#7
 is($retval, 0, "return value should be equal zero: \"$retval\"");
 $tests++;
 

t/02_clientsim_txt_parsable.t

@@ -14,9 +14,12 @@ my $error_regexp2='client-simulation.txt:';
 
 printf "\n%s\n", "Testing whether \"~/etc/client-simulation.txt\" isn't broken ...";
 $fileout = `bash ./etc/client-simulation.txt 2>&1`;
+
+#1
 unlike($fileout, qr/$error_regexp1/, "regex 1");
 $tests++;
 
+#2
 unlike($fileout, qr/$error_regexp2/, "regex 2");
 $tests++;
 

t/05_ca_hashes_up_to_date.t

@@ -3,10 +3,12 @@
 use strict;
 use Test::More;
 
-printf "\n%s\n", "Testing whether CA certificates are newer their SPKI hashes \"~/etc/ca_hashes.txt\" ...";
+printf "\n%s\n", "Make sure CA certificate stores are older than their SPKI hashes \"~/etc/ca_hashes.txt\" ...";
 
 my $newer_bundles=`find etc/*.pem -newer etc/ca_hashes.txt`;
-is($newer_bundles,"","Checking if there's an output with a *.pem file run \"~/utils/create_ca_hashes.sh\"");
+
+#1
+is($newer_bundles,"","Checking if there's an output with a *.pem file. If so: run \"~/utils/create_ca_hashes.sh\"");
 
 printf "\n";
 done_testing;

t/10_baseline_ipv4_http.t

@@ -22,7 +22,7 @@ my $socket_out="";
 my $openssl_out="";
 my $socket_json="";
 my $openssl_json="";
-#FIXME: Blacklists we use to trigger an error, but likely we can skip that and instead we should?/could use
+#FIXME: Pattern we use to trigger an error, but likely we can skip that and instead we should?/could use the following??
 #       @args="$prg $check2run $uri >/dev/null";
 #       system("@args") == 0
 #           or die ("FAILED: \"@args\" ");
@@ -38,23 +38,26 @@ unlink $tmp_json;
 
 # Title
 printf "\n%s\n", "Baseline unit test IPv4 against \"$uri\"";
-
-#1
 $socket_out = `$prg $check2run $uri 2>&1`;
 $socket_json = json($tmp_json);
+
+#1
 unlike($socket_out, qr/$socket_errors≈/, "via sockets, checking terminal output");
 $tests++;
+
+#2
 unlike($socket_json, qr/$json_errors/, "via sockets checking JSON output");
 $tests++;
 
 unlink $tmp_json;
 
-
+#3
-#2
 $openssl_out = `$prg --ssl-native $check2run $uri 2>&1`;
 $openssl_json = json($tmp_json);
 unlike($openssl_out, qr/$openssl_errors/, "via (builtin) OpenSSL, checking terminal output");
 $tests++;
+
+#4
 unlike($openssl_json, qr/$json_errors/, "via OpenSSL (builtin) checking JSON output");
 $tests++;
 

t/21_baseline_starttls.t

@@ -35,14 +35,17 @@ die "Unable to open $prg" unless -f $prg;
 
 $uri="smtp-relay.gmail.com:587";
 
-# we will have client simulations later, so we don't need to run everything again:
+
+#1
 # unlink "tmp.json";
+# we will have client simulations later, so we don't need to run everything again:
 printf "\n%s\n", "STARTTLS SMTP unit test via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run_smtp -t smtp $uri 2>&1`;
 # $socket_json = json('tmp.json');
 unlike($socket_out, qr/$socket_regex_bl/, "");
 $tests++;
 
+#2
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS SMTP unit tests via OpenSSL --> $uri ...";
 $openssl_out = `./testssl.sh --ssl-native $check2run_smtp -t smtp $uri 2>&1`;
@@ -50,9 +53,9 @@ $openssl_out = `./testssl.sh --ssl-native $check2run_smtp -t smtp $uri 2>&1`;
 unlike($openssl_out, qr/$openssl_regex_bl/, "");
 $tests++;
 
-
 $uri="pop.gmx.net:110";
 
+#3
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS POP3 unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t pop3 $uri 2>&1`;
@@ -60,15 +63,16 @@ $socket_out = `./testssl.sh $check2run -t pop3 $uri 2>&1`;
 unlike($socket_out, qr/$socket_regex_bl/, "");
 $tests++;
 
+#4
 printf "\n%s\n", "STARTTLS POP3 unit tests via OpenSSL --> $uri ...";
 $openssl_out = `./testssl.sh --ssl-native $check2run -t pop3 $uri 2>&1`;
 # $openssl_json = json('tmp.json');
 unlike($openssl_out, qr/$openssl_regex_bl/, "");
 $tests++;
 
-
 $uri="imap.gmx.net:143";
 
+#5
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS IMAP unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t imap $uri 2>&1`;
@@ -76,15 +80,16 @@ $socket_out = `./testssl.sh $check2run -t imap $uri 2>&1`;
 unlike($socket_out, qr/$socket_regex_bl/, "");
 $tests++;
 
+#6
 printf "\n%s\n", "STARTTLS IMAP unit tests via OpenSSL --> $uri ...";
 $openssl_out = `./testssl.sh --ssl-native $check2run -t imap $uri 2>&1`;
 # $openssl_json = json('tmp.json');
 unlike($openssl_out, qr/$openssl_regex_bl/, "");
 $tests++;
 
-
 $uri="mail.tigertech.net:4190";
 
+#7
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS MANAGE(SIEVE) unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t sieve $uri 2>&1`;
@@ -92,9 +97,9 @@ $socket_out = `./testssl.sh $check2run -t sieve $uri 2>&1`;
 unlike($openssl_out, qr/$openssl_regex_bl/, "");
 $tests++;
 
-
 $uri="jabber.org:5222";
 
+#8
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS XMPP unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t xmpp $uri 2>&1`;
@@ -120,6 +125,7 @@ $tests++;
 
 $uri="ldap.uni-rostock.de:21";
 
+#9
 # unlink "tmp.json";
 printf "\n%s\n", "STARTTLS FTP unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t ftp $uri 2>&1`;
@@ -143,12 +149,14 @@ $tests++;
 # https://ldapwiki.com/wiki/Public%20LDAP%20Servers
 $uri="db.debian.org:389";
 
+#10
 printf "\n%s\n", "STARTTLS LDAP unit tests via sockets --> $uri ...";
 $socket_out = `./testssl.sh $check2run -t ldap $uri 2>&1`;
 # $socket_json = json('tmp.json');
 unlike($socket_out, qr/$socket_regex_bl/, "");
 $tests++;
 
+#11
 printf "\n%s\n", "STARTTLS LDAP unit tests via OpenSSL --> $uri ...";
 $openssl_out = `./testssl.sh --ssl-native $check2run -t ldap $uri 2>&1`;
 # $openssl_json = json('tmp.json');
@@ -172,12 +180,10 @@ $tests++;
 #unlike($openssl_out, qr/$openssl_regex_bl/, "");
 #$tests++;
 
-
 # IRC: missing
 # LTMP, mysql, postgres
 
 
-
 done_testing($tests);
 # unlink "tmp.json";
 

t/31_isJSON_valid.t

@@ -13,9 +13,12 @@ my $check2run ="--ip=one --ids-friendly -q --color 0";
 my $uri="";
 my $json="";
 my $out="";
+my $cmd_timeout="--openssl-timeout=10";
 # Blacklists we use to trigger an error:
 my $socket_regex_bl='(e|E)rror|\.\/testssl\.sh: line |(f|F)atal|(c|C)ommand not found';
 my $openssl_regex_bl='(e|E)rror|(f|F)atal|\.\/testssl\.sh: line |Oops|s_client connect problem|(c|C)ommand not found';
+# that can be done better but I am a perl n00b ;-)
+my $os=`perl -e 'print "$^O";'`;
 
 die "Unable to open $prg" unless -f $prg;
 
@@ -33,6 +36,7 @@ my @errors=eval { decode_json($json) };
 is(@errors,0,"no errors");
 $tests++;
 
+
 #2
 printf "%s\n", ".. pretty JSON --> $uri ";
 $out = `./testssl.sh $check2run --jsonfile-pretty tmp.json $uri`;
@@ -44,7 +48,21 @@ $tests++;
 
 
 #3
-# This testssl.sh run deliberately does NOT work as travis-ci.org blocks port 25 egress.
+my $uri = "smtp-relay.gmail.com:587";
+printf "%s\n", " .. plain JSON and STARTTLS --> $uri ...";
+$out = `./testssl.sh  --jsonfile tmp.json $check2run -t smtp $uri`;
+$json = json('tmp.json');
+unlink 'tmp.json';
+@errors=eval { decode_json($json) };
+is(@errors,0,"no errors");
+$tests++;
+
+if ( $os eq "linux" ){
+     # macos doesn't have a timeout command, unless we install coreutils (gnu coreutils)
+     # so we just silently skip this
+
+     #4
+     # This testssl.sh run deliberately does NOT work as github actions block port 25 egress.
      # but the output should be fine. The idea is to have a unit test for a failed connection.
      printf "%s\n", ".. plain JSON for a failed run: '--mx $uri' ...";
      $out = `./testssl.sh --ssl-native --openssl-timeout=10 $check2run --jsonfile tmp.json --mx $uri`;
@@ -54,7 +72,7 @@ unlink 'tmp.json';
      is(@errors,0,"no errors");
      $tests++;
 
-#4
+     #5
      # Same as above but with pretty JSON
      printf "%s\n", ".. pretty JSON for a failed run '--mx $uri' ...";
      $out = `./testssl.sh --ssl-native --openssl-timeout=10 $check2run --jsonfile-pretty tmp.json --mx $uri`;
@@ -64,15 +82,9 @@ unlink 'tmp.json';
      is(@errors,0,"no errors");
      $tests++;
 
-#5
+} elsif ( $os eq "darwin" ){
-my $uri = "smtp-relay.gmail.com:587";
+     printf "skipped two checks on MacOS\n\n";
-printf "%s\n", " .. plain JSON and STARTTLS --> $uri ...";
+}
-$out = `./testssl.sh  --jsonfile tmp.json $check2run -t smtp $uri`;
-$json = json('tmp.json');
-unlink 'tmp.json';
-@errors=eval { decode_json($json) };
-is(@errors,0,"no errors");
-$tests++;
 
 printf "\n";
 done_testing($tests);

t/61_diff_testsslsh.t

@@ -50,6 +50,24 @@ $baseline_csv =~ s/HTTP_headerTime.*\n//g;
 $cat_csv      =~ s/censys.io.*\n//g;
 $baseline_csv =~ s/censys.io.*\n//g;
 
+# MacOS / LibreSSL has different OpenSSL names for TLS 1.3 ciphers. That should be rather solved in
+#       testssl.sh, see #2763. But for now we do this here.
+$cat_csv      =~ s/AEAD-AES128-GCM-SHA256/TLS_AES_128_GCM_SHA256/g;
+$cat_csv      =~ s/AEAD-AES256-GCM-SHA384/TLS_AES_256_GCM_SHA384/g;
+# this is a bit ugly but otherwise the line cipher-tls1_3_x1303 with the CHACHA20 cipher misses a space
+$cat_csv      =~ s/x1303   AEAD-CHACHA20-POLY1305-SHA256/x1303   TLS_CHACHA20_POLY1305_SHA256 /g;
+# now the other lines, where we don't need to insert the additional space:
+$cat_csv      =~ s/AEAD-CHACHA20-POLY1305-SHA256/TLS_CHACHA20_POLY1305_SHA256/g;
+
+# Same with ECDH bit length
+$cat_csv      =~ s/ECDH 253/ECDH 256/g;
+$baseline_csv =~ s/ECDH 253/ECDH 256/g;
+
+
+# this could contain the openssl path
+$cat_csv      =~ s/"engine_problem.*\n//g;
+$baseline_csv =~ s/"engine_problem.*\n//g;
+
 $diff = diff \$cat_csv, \$baseline_csv;
 
 # Compare the differences to the baseline file -- and print differences if there were detected.

testssl.sh

@@ -17665,7 +17665,7 @@ run_renego() {
                          && [[ $(tail -1 $ERRFILE | grep -acE '^(RENEGOTIATING|depth|verify|notAfter)') -eq 1 ]] \
                          && [[ $k -lt 120 ]]; \
                        do sleep $ssl_reneg_wait; ((k++)); if (tail -5 $TMPFILE| grep -qa '^closed'); then break; fi; done; \
-               done) | \
+               done) 2> /dev/null | \
                $OPENSSL_NOTIMEOUT s_client $(s_client_options "$proto $legacycmd $STARTTLS $BUGS -connect $NODEIP:$PORT $PROXY $SNI") >$TMPFILE 2>$ERRFILE &
           pid=$!
           ( sleep $((ssl_reneg_attempts*3+3)) && kill $pid && touch $TEMPDIR/was_killed ) >&2 2>/dev/null &