Merge pull request #2639 from testssl/fix_ci_runs

Fix ci runs

t/10_baseline_ipv4_http.t

@@ -15,48 +15,51 @@ use JSON;
 
 my $tests = 0;
 my $prg="./testssl.sh";
-my $check2run="-p -s -P --fs -S -h -U -q --ip=one --color 0";
+my $tmp_json="tmp.json";
+my $check2run="-p -s -P --fs -S -h -U -q --ip=one --color 0 --jsonfile $tmp_json";
 my $uri="google.com";
 my $socket_out="";
 my $openssl_out="";
-# Blacklists we use to trigger an error:
-my $socket_regex_bl='(e|E)rror|\.\/testssl\.sh: line |(f|F)atal|(c|C)ommand not found';
-my $openssl_regex_bl='(e|E)rror|(f|F)atal|\.\/testssl\.sh: line |Oops|s_client connect problem|(c|C)ommand not found';
-my $json_regex_bl='(id".*:\s"scanProblem"|severity".*:\s"FATAL"|"Scan interrupted")';
-
 my $socket_json="";
 my $openssl_json="";
-$check2run="--jsonfile tmp.json $check2run";
+#FIXME: Blacklists we use to trigger an error, but likely we can skip that and instead we should?/could use
+#       @args="$prg $check2run $uri >/dev/null";
+#       system("@args") == 0
+#           or die ("FAILED: \"@args\" ");
+my $socket_errors='(e|E)rror|\.\/testssl\.sh: line |(f|F)atal|(c|C)ommand not found';
+my $openssl_errors='(e|E)rror|(f|F)atal|\.\/testssl\.sh: line |Oops|s_client connect problem|(c|C)ommand not found';
+my $json_errors='(id".*:\s"scanProblem"|severity".*:\s"FATAL"|"Scan interrupted")';
+
 
 die "Unable to open $prg" unless -f $prg;
 
 # Provide proper start conditions
-unlink "tmp.json";
+unlink $tmp_json;
 
 # Title
 printf "\n%s\n", "Baseline unit test IPv4 against \"$uri\"";
 
 #1
 $socket_out = `$prg $check2run $uri 2>&1`;
-$socket_json = json('tmp.json');
+$socket_json = json($tmp_json);
-unlink "tmp.json";
+unlike($socket_out, qr/$socket_errors≈/, "via sockets, checking terminal output");
-unlike($socket_out, qr/$socket_regex_bl/, "via sockets, terminal output");
 $tests++;
-unlike($socket_json, qr/$json_regex_bl/, "via sockets JSON output");
+unlike($socket_json, qr/$json_errors/, "via sockets checking JSON output");
 $tests++;
 
+unlink $tmp_json;
+
+
 #2
 $openssl_out = `$prg --ssl-native $check2run $uri 2>&1`;
-$openssl_json = json('tmp.json');
+$openssl_json = json($tmp_json);
-unlink "tmp.json";
+unlike($openssl_out, qr/$openssl_errors/, "via (builtin) OpenSSL, checking terminal output");
-# With Google only we sometimes encounter an error as they return a 0 char with openssl, so we white list this pattern here:
-# It should be fixed in the code though so we comment this out
-# $openssl_out =~ s/testssl.*warning: command substitution: ignored null byte in input\n//g;
-unlike($openssl_out, qr/$openssl_regex_bl/, "via OpenSSL");
 $tests++;
-unlike($openssl_json, qr/$json_regex_bl/, "via OpenSSL JSON output");
+unlike($openssl_json, qr/$json_errors/, "via OpenSSL (builtin) checking JSON output");
 $tests++;
 
+unlink $tmp_json;
+
 done_testing($tests);
 printf "\n";
 
@@ -69,5 +72,5 @@ sub json($) {
 }
 
 
-#  vim:ts=5:sw=5:expandtab
+# vim:ts=5:sw=5:expandtab
 

t/12_diff_opensslversions.t

@@ -25,18 +25,18 @@ die "Unable to open $prg" unless -f $prg;
 die "Unable to open $distro_openssl" unless -f $distro_openssl;
 
 # Provide proper start conditions
-unlink "tmp.csv";
+unlink $csvfile;
-unlink "tmp2.csv";
+unlink $csvfile2;
 
 #1 run
 printf "\n%s\n", "Diff test IPv4 with supplied openssl against \"$uri\"";
-@args="$prg $check2run $csvfile $uri 2>&1";
+@args="$prg $check2run $csvfile $uri >/dev/null";
 system("@args") == 0
      or die ("FAILED: \"@args\"");
 
 # 2
 printf "\n%s\n", "Diff test IPv4 with $distro_openssl against \"$uri\"";
-@args="$prg $check2run $csvfile2 --openssl=$distro_openssl $uri 2>&1";
+@args="$prg $check2run $csvfile2 --openssl=$distro_openssl $uri >/dev/null";
 system("@args") == 0
      or die ("FAILED: \"@args\" ");
 
@@ -63,6 +63,10 @@ $cat_csvfile  =~  s/ECDH\/MLKEM/ECDH 253  /g;
 $cat_csvfile  =~ s/.nonce-.* //g;
 $cat_csvfile2 =~ s/.nonce-.* //g;
 
+# Fix IP addresses. Needed when we don't hit the same IP address. We just remove them
+$cat_csvfile  =~ s/","google.com\/.*","443/","google.com","443/g;
+$cat_csvfile2 =~ s/","google.com\/.*","443/","google.com","443/g;
+
 $diff = diff \$cat_csvfile, \$cat_csvfile2;
 
 # Compare the differences -- and print them if there were any

t/61_diff_testsslsh.t

@@ -20,6 +20,7 @@ my $cat_csv="tmp.csv";
 my $check2run="-p -s -P --fs -h -U -c -q --ip=one --color 0 --csvfile $cat_csv";
 my $uri="testssl.sh";
 my $diff="";
+my @args="";
 
 die "Unable to open $prg" unless -f $prg;
 die "Unable to open $baseline_csv" unless -f $baseline_csv;
@@ -27,11 +28,10 @@ die "Unable to open $baseline_csv" unless -f $baseline_csv;
 # Provide proper start conditions
 unlink $cat_csv;
 
-my @args=("$prg", "$check2run", "$uri", "2>&1");
 
 #1 run
 printf "\n%s\n", "Diff unit test (IPv4) against \"$uri\"";
-printf "@args\n";
+@args="$prg $check2run $uri >/dev/null";
 system("@args") == 0
      or die ("FAILED: \"@args\" ");
 

t/Readme.md

@@ -5,6 +5,6 @@
 * 30-39:  Does reporting work?
 * 50-69:  Are the results what I expect (server side)?
 
-Please help to write Travis/CI tests! Documentation can be found [here](https://perldoc.perl.org/Test/More.html).
+Please help to write CI tests! Documentation can be found [here](https://perldoc.perl.org/Test/More.html).
-You can consult the existing code here. Feel free to use `10_baseline_ipv4_http.t` or `23_client_simulation.t` as a
+You can consult the existing code here. Feel free to use `10_baseline_ipv4_http.t` or `12_diff_opensslversions.t` as a
-template.
+template. The latter is newer and code is cleaner.