Merge pull request #1262 from drwetter/more_unittests1

More unit / integration tests + Fix client simulation with OpenSSL, LDAP

doc/testssl.1

@@ -122,7 +122,7 @@ Please note that \fBfname\fR has to be in Unix format\. DOS carriage returns won
 \fB\-\-mode <serial|parallel>\fR\. Mass testing to be done serial (default) or parallel (\fB\-\-parallel\fR is shortcut for the latter, \fB\-\-serial\fR is the opposite option)\. Per default mass testing is being run in serial mode, i\.e\. one line after the other is processed and invoked\. The variable \fBMASS_TESTING_MODE\fR can be defined to be either equal \fBserial\fR or \fBparallel\fR\.
 .
 .SS "SPECIAL INVOCATIONS"
-\fB\-t <protocol>, \-\-starttls <protocol>\fR does a default run against a STARTTLS enabled \fBprotocol\fR\. \fBprotocol\fR must be one of \fBftp\fR, \fBsmtp\fR, \fBpop3\fR, \fBimap\fR, \fBxmpp\fR, \fBtelnet\fR, \fBldap\fR, \fBlmtp\fR, \fBnntp\fR, \fBpostgres\fR, \fBmysql\fR\. For the latter four you need e\.g\. the supplied OpenSSL or OpenSSL version 1\.1\.1\. Please note: MongoDB doesn\'t offer a STARTTLS connection\.
+\fB\-t <protocol>, \-\-starttls <protocol>\fR does a default run against a STARTTLS enabled \fBprotocol\fR\. \fBprotocol\fR must be one of \fBftp\fR, \fBsmtp\fR, \fBpop3\fR, \fBimap\fR, \fBxmpp\fR, \fBtelnet\fR, \fBldap\fR, \fBlirc\fR, \fBlmtp\fR, \fBnntp\fR, \fBpostgres\fR, \fBmysql\fR\. For the latter four you need e\.g\. the supplied OpenSSL or OpenSSL version 1\.1\.1\. Please note: MongoDB doesn\'t offer a STARTTLS connection, LDAP currently only works with \fB--ssl-native\fR\. \fBtelnet\fR and \fBirc\fR is WIP\.
 .
 .P
 \fB\-\-xmpphost <jabber_domain>\fR is an additional option for STARTTLS enabled XMPP: It expects the jabber domain as a parameter\. This is only needed if the domain is different from the URI supplied\.
@@ -323,7 +323,7 @@ Security headers (X\-Frame\-Options, X\-XSS\-Protection, Expect\-CT,\.\.\. , CSP
 \fB\-g, \-\-grease\fR checks several server implementation bugs like tolerance to size limitations and GREASE, see https://www\.ietf\.org/archive/id/draft\-ietf\-tls\-grease\-01\.txt \. This checks doesn\'t run per default\.
 .
 .SS "VULNERABILITIES"
-\fB\-U, \-\-vulnerable\fR Just tests all (of the following) vulnerabilities\. The environment variable \fBVULN_THRESHLD\fR determines after which value a separate headline for each vulnerability is being displayed\. Default is \fB1\fR which means if you check for two vulnerabilities, only the general headline for vulnerabilities section is displayed \-\- in addition to the vulnerability and the result\. Otherwise each vulnerability or vulnerability section gets its own headline in addition to the output of the name of the vulnerabilty and test result\. A vulnerability section is comprised of more than one check, e\.g\. the renegotiation vulnerability check has two checks, so has Logjam\.
+\fB\-U, \-\-vulnerable, \-\-vulnerablilities\fR Just tests all (of the following) vulnerabilities\. The environment variable \fBVULN_THRESHLD\fR determines after which value a separate headline for each vulnerability is being displayed\. Default is \fB1\fR which means if you check for two vulnerabilities, only the general headline for vulnerabilities section is displayed \-\- in addition to the vulnerability and the result\. Otherwise each vulnerability or vulnerability section gets its own headline in addition to the output of the name of the vulnerabilty and test result\. A vulnerability section is comprised of more than one check, e\.g\. the renegotiation vulnerability check has two checks, so has Logjam\.
 .
 .P
 \fB\-H, \-\-heartbleed\fR Checks for Heartbleed, a memory leakage in openssl\. Unless the server side doesn\'t support the heartbeat extension it is likely that this check runs into a timeout\. The seconds to wait for a reply can be adjusted with \fBHEARTBLEED_MAX_WAITSOCK\fR\. 8 is the default\.

doc/testssl.1.html

@@ -180,7 +180,7 @@ host.example.com:631
 
 <h3 id="SPECIAL-INVOCATIONS">SPECIAL INVOCATIONS</h3>
 
-<p><code>-t &lt;protocol>, --starttls &lt;protocol></code>    does a default run against a STARTTLS enabled <code>protocol</code>. <code>protocol</code> must be one of <code>ftp</code>, <code>smtp</code>,  <code>pop3</code>, <code>imap</code>, <code>xmpp</code>, <code>telnet</code>, <code>ldap</code>, <code>lmtp</code>, <code>nntp</code>, <code>postgres</code>, <code>mysql</code>. For the latter four you need e.g. the supplied OpenSSL or OpenSSL version 1.1.1. Please note: MongoDB doesn't offer a STARTTLS connection.</p>
+<p><code>-t &lt;protocol>, --starttls &lt;protocol></code>    does a default run against a STARTTLS enabled <code>protocol</code>. <code>protocol</code> must be one of <code>ftp</code>, <code>smtp</code>,  <code>pop3</code>, <code>imap</code>, <code>xmpp</code>, <code>telnet</code>, <code>ldap</code>, <code>irc</code>, <code>lmtp</code>, <code>nntp</code>, <code>postgres</code>, <code>mysql</code>. For the latter four you need e.g. the supplied OpenSSL or OpenSSL version 1.1.1. Please note: MongoDB doesn't offer a STARTTLS connection.  LDAP currently only works with <code>--ssl-native</code>. <code>telnet</code> and <code>irc</code> is WIP</p>
 
 <p><code>--xmpphost &lt;jabber_domain></code> is an additional option for STARTTLS enabled XMPP: It expects the jabber domain as a parameter. This is only needed if the domain is different from the URI supplied.</p>
 
@@ -297,7 +297,7 @@ Also for multiple server certificates are being checked for as well as for the c
 
 <h3 id="VULNERABILITIES">VULNERABILITIES</h3>
 
-<p><code>-U, --vulnerable</code>              Just tests all (of the following) vulnerabilities. The environment variable <code>VULN_THRESHLD</code> determines after which value a separate headline for each vulnerability is being displayed. Default is <code>1</code> which means if you check for two vulnerabilities, only the general headline for vulnerabilities section is displayed -- in addition to the vulnerability and the result. Otherwise each vulnerability or vulnerability section gets its own headline in addition to the output of the name of the vulnerabilty and test result. A vulnerability section is comprised of more than one check, e.g. the renegotiation vulnerability check has two checks, so has Logjam.</p>
+<p><code>-U, --vulnerable, --vulnerabilities</code>              Just tests all (of the following) vulnerabilities. The environment variable <code>VULN_THRESHLD</code> determines after which value a separate headline for each vulnerability is being displayed. Default is <code>1</code> which means if you check for two vulnerabilities, only the general headline for vulnerabilities section is displayed -- in addition to the vulnerability and the result. Otherwise each vulnerability or vulnerability section gets its own headline in addition to the output of the name of the vulnerabilty and test result. A vulnerability section is comprised of more than one check, e.g. the renegotiation vulnerability check has two checks, so has Logjam.</p>
 
 <p><code>-H, --heartbleed</code>              Checks for Heartbleed, a memory leakage in openssl. Unless the server side doesn't support the heartbeat extension it is likely that this check runs into a timeout. The seconds to wait for a reply can be adjusted with <code>HEARTBLEED_MAX_WAITSOCK</code>. 8 is the default.</p>
 

doc/testssl.1.md

@@ -101,7 +101,7 @@ Please note that `fname` has to be in Unix format. DOS carriage returns won't be
 
 ### SPECIAL INVOCATIONS
 
-`-t <protocol>, --starttls <protocol>`    does a default run against a STARTTLS enabled `protocol`. `protocol` must be one of `ftp`, `smtp`,  `pop3`, `imap`, `xmpp`, `telnet`, `ldap`, `lmtp`, `nntp`, `postgres`, `mysql`. For the latter four you need e.g. the supplied OpenSSL or OpenSSL version 1.1.1. Please note: MongoDB doesn't offer a STARTTLS connection.
+`-t <protocol>, --starttls <protocol>`    does a default run against a STARTTLS enabled `protocol`. `protocol` must be one of `ftp`, `smtp`,  `pop3`, `imap`, `xmpp`, `telnet`, `ldap`, `irc`, `lmtp`, `nntp`, `postgres`, `mysql`. For the latter four you need e.g. the supplied OpenSSL or OpenSSL version 1.1.1. Please note: MongoDB doesn't offer a STARTTLS connection, LDAP currently only works with `--ssl-native`. `telnet` and `irc` is WIP.
 
 `--xmpphost <jabber_domain>` is an additional option for STARTTLS enabled XMPP: It expects the jabber domain as a parameter. This is only needed if the domain is different from the URI supplied.
 
@@ -210,7 +210,7 @@ Also for multiple server certificates are being checked for as well as for the c
 
 ### VULNERABILITIES
 
-`-U, --vulnerable`              Just tests all (of the following) vulnerabilities. The environment variable `VULN_THRESHLD` determines after which value a separate headline for each vulnerability is being displayed. Default is `1` which means if you check for two vulnerabilities, only the general headline for vulnerabilities section is displayed -- in addition to the vulnerability and the result. Otherwise each vulnerability or vulnerability section gets its own headline in addition to the output of the name of the vulnerabilty and test result. A vulnerability section is comprised of more than one check, e.g. the renegotiation vulnerability check has two checks, so has Logjam.
+`-U, --vulnerable, --vulnerabilities` Just tests all (of the following) vulnerabilities. The environment variable `VULN_THRESHLD` determines after which value a separate headline for each vulnerability is being displayed. Default is `1` which means if you check for two vulnerabilities, only the general headline for vulnerabilities section is displayed -- in addition to the vulnerability and the result. Otherwise each vulnerability or vulnerability section gets its own headline in addition to the output of the name of the vulnerabilty and test result. A vulnerability section is comprised of more than one check, e.g. the renegotiation vulnerability check has two checks, so has Logjam.
 
 `-H, --heartbleed`              Checks for Heartbleed, a memory leakage in openssl. Unless the server side doesn't support the heartbeat extension it is likely that this check runs into a timeout. The seconds to wait for a reply can be adjusted with `HEARTBLEED_MAX_WAITSOCK`. 8 is the default.
 

t/20_baseline_ipv4_http.t

@@ -0,0 +1,54 @@
+#!/usr/bin/env perl
+
+# Just a functional test, whether there are any problems on the client side
+# Probably we could also inspect the JSON for any problems for
+#    "id"           : "scanProblem"
+#    "finding"      : "Scan interrupted"
+
+use strict;
+use Test::More;
+use Data::Dumper;
+# use JSON;
+
+my $tests = 0;
+my $check2run ="-p -s -P --pfs -S -h -U -q --ip=one --color 0";
+my $uri="";
+my $socketout="";
+my $opensslout="";
+
+# $check2run="--jsonfile tmp.json $check2run";
+
+
+$uri="google.com";
+
+unlink "tmp.json";
+printf "\n%s\n", "Baseline unit test IPv4 via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run $uri 2>&1`;
+# my $socket = json('tmp.json');
+unlike($socketout, qr/(e|E)rror|\.\/testssl\.sh: line |(f|F)atal/, "");
+$tests++;
+
+
+unlink "tmp.json";
+printf "\n%s\n", "Baseline unit test IPv4 via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh $check2run --ssl-native $uri 2>&1`;
+# my $openssl = json('tmp.json');
+# This happens with Google only, so we white list a pattern here:
+$opensslout =~ s/testssl.*warning: command substitution: ignored null byte in input\n//g;
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|\.\/testssl\.sh: line |Oops|s_client connect problem/, "");
+$tests++;
+
+
+
+done_testing($tests);
+unlink "tmp.json";
+
+
+
+sub json($) {
+	my $file = shift;
+	$file = `cat $file`;
+	unlink $file;
+	return from_json($file);
+}
+

t/21_baseline_ipv6_http.t.DISABLED

@@ -0,0 +1,52 @@
+#!/usr/bin/env perl
+
+# disabled as IPv6 is not supported by Travis, see https://github.com/drwetter/testssl.sh/issues/1177
+
+# Just a functional test, whether there are any problems on the client side
+# Probably we could also inspect the JSON for any problems for
+#    "id"           : "scanProblem"
+#    "finding"      : "Scan interrupted"
+
+use strict;
+use Test::More;
+use Data::Dumper;
+# use JSON;
+
+my $tests = 0;
+my $check2run ="-p -s -P --pfs -S -h -U -q --ip=one --color 0";
+my $uri="";
+my $socketout="";
+my $opensslout="";
+
+# $check2run="--jsonfile tmp.json $check2run";
+
+
+$uri="testssl.net";
+
+unlink "tmp.json";
+printf "\n%s\n", "Baseline unit test IPv6 via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run -6 $uri 2>&1`;
+# my $socket = json('tmp.json');
+unlike($socketout, qr/(e|E)rror|\.\/testssl\.sh: line |(f|F)atal/, "");
+$tests++;
+
+unlink "tmp.json";
+printf "\n%s\n", "Baseline unit test IPv6 via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run -6 $uri 2>&1`;
+# my $openssl = json('tmp.json');
+$opensslout =~ s/testssl.*warning: command substitution: ignored null byte in input\n//g;
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|\.\/testssl\.sh: line |Oops|s_client connect problem/, "");
+$tests++;
+
+done_testing($tests);
+unlink "tmp.json";
+
+
+
+sub json($) {
+	my $file = shift;
+	$file = `cat $file`;
+	unlink $file;
+	return from_json($file);
+}
+

t/23_client_simulation.t

@@ -0,0 +1,69 @@
+#!/usr/bin/env perl
+
+# Just a functional test, whether there are any problems on the client side
+# Probably we could also inspect the JSON for any problems for
+#    "id"           : "scanProblem"
+#    "finding"      : "Scan interrupted"
+
+use strict;
+use Test::More;
+use Data::Dumper;
+use JSON;
+
+my $tests = 0;
+my $check2run ="--client-simulation -q --ip=one --color 0";
+my $uri="";
+my $socketout="";
+my $opensslout="";
+
+# $check2run="--jsonfile tmp.json $check2run";
+
+
+$uri="google.com";
+
+unlink "tmp.json";
+printf "\n%s\n", "Client simulations unit test via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run $uri`;
+# my $socket = json('tmp.json');
+#FIXME: This comparison is maybe not sufficient yet:
+unlike($socketout, qr/(e|E)rror|(f|F)atal/, "");
+$tests++;
+
+
+unlink "tmp.json";
+printf "\n%s\n", "Client simulations unit test via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh $check2run --ssl-native $uri`;
+# my $openssl = json('tmp.json');
+#FIXME: This comparison is maybe sufficient yet:
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+
+$uri="smtp-relay.gmail.com:587";
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS: Client simulations unit test via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run -t smtp $uri`;
+# my $socket = json('tmp.json');
+unlike($socketout, qr/(e|E)rror|(f|F)atal/, "");
+$tests++;
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS: Client simulations unit test via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run -t smtp $uri`;
+# my $openssl = json('tmp.json');
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+done_testing($tests);
+unlink "tmp.json";
+
+
+
+sub json($) {
+	my $file = shift;
+	$file = `cat $file`;
+	unlink $file;
+	return from_json($file);
+}
+

t/25_baseline_starttls.t

@@ -0,0 +1,154 @@
+#!/usr/bin/env perl
+
+# Just a functional test, whether there are any problems on the client side
+# Probably we could also inspect the JSON for any problems for 
+#    "id"           : "scanProblem"
+#    "finding"      : "Scan interrupted"
+
+# Catches:
+# - This unit test takes very long
+# - Hosts which match the regex patterns should be avoided
+
+use strict;
+use Test::More;
+use Data::Dumper;
+use JSON;
+
+my $tests = 0;
+my $check2run_smtp="--protocols --standard --pfs --server-preference --headers --vulnerable --each-cipher -q --ip=one --color 0";
+my $check2run="-q --ip=one --color 0";
+my $uri="";
+my $socketout="";
+my $opensslout="";
+
+# $check2run_smtp="--jsonfile tmp.json $check2run_smtp";
+# $check2run="--jsonfile tmp.json $check2run";
+
+$uri="smtp-relay.gmail.com:587";
+
+# we will have client simulations later, so we don't need to run everything again:
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS SMTP unit test via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run_smtp -t smtp $uri`;
+# my $socket = json('tmp.json');
+unlike($socketout, qr/(e|E)rror|(f|F)atal/, "");
+$tests++;
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS SMTP unit tests via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run_smtp -t smtp $uri`;
+# my $openssl = json('tmp.json');
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+
+$uri="pop.gmx.net:110";
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS POP3 unit tests via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run -t pop3 $uri`;
+# my $socket = json('tmp.json');
+unlike($socketout, qr/(e|E)rror|(f|F)atal/, "");
+$tests++;
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS POP3 unit tests via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run -t pop3 $uri`;
+# my $openssl = json('tmp.json');
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+
+$uri="imap.gmx.net:143";
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS IMAP unit tests via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run -t imap $uri`;
+# my $socket = json('tmp.json');
+unlike($socketout, qr/(e|E)rror|(f|F)atal/, "");
+$tests++;
+
+printf "\n%s\n", "STARTTLS IMAP unit tests via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run -t imap $uri`;
+# my $openssl = json('tmp.json');
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+
+$uri="jabber.org:5222";
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS XMPP unit tests via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run -t xmpp $uri`;
+# my $socket = json('tmp.json');
+unlike($socketout, qr/(e|E)rror|(f|F)atal/, "");
+$tests++;
+
+printf "\n%s\n", "STARTTLS XMPP unit tests via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run -t xmpp $uri`;
+# my $openssl = json('tmp.json');
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+
+$uri="ldap.uni-rostock.de:21";
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS FTP unit tests via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run -t ftp $uri`;
+# my $socket = json('tmp.json');
+# OCSP stapling fails sometimes with: 'offered, error querying OCSP responder (ERROR: No Status found)'
+$socketout =~ s/ error querying OCSP responder .*\n//g;
+unlike($socketout, qr/(e|E)rror|(f|F)atal/, "");
+$tests++;
+
+printf "\n%s\n", "STARTTLS FTP unit tests via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run -t ftp $uri`;
+# my $openssl = json('tmp.json');
+# OCSP stapling fails sometimes with: 'offered, error querying OCSP responder (ERROR: No Status found)'
+$opensslout =~ s/ error querying OCSP responder .*\n//g;
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+
+# https://ldapwiki.com/wiki/Public%20LDAP%20Servers
+$uri="ldap.telesec.de:389";
+
+printf "\n%s\n", "STARTTLS LDAP unit tests via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run -t ldap $uri`;
+# my $openssl = json('tmp.json');
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+
+$uri="news.newsguy.com:119";
+
+unlink "tmp.json";
+printf "\n%s\n", "STARTTLS NNTP unit tests via sockets --> $uri ...";
+$socketout = `./testssl.sh $check2run -t nntp $uri`;
+# my $socket = json('tmp.json');
+unlike($socketout, qr/(e|E)rror|(f|F)atal/, "");
+$tests++;
+
+printf "\n%s\n", "STARTTLS NNTP unit tests via OpenSSL --> $uri ...";
+$opensslout = `./testssl.sh --ssl-native $check2run -t nntp $uri`;
+# my $openssl = json('tmp.json');
+unlike($opensslout, qr/(e|E)rror|(f|F)atal|Oops|s_client connect problem/, "");
+$tests++;
+
+
+# IRC: missing
+# LTMP, mysql, postgres
+
+
+
+done_testing($tests);
+unlink "tmp.json";
+
+sub json($) {
+	my $file = shift;
+	$file = `cat $file`;
+	unlink $file;
+	return from_json($file);
+}
+

t/25_client_sims_starttls.t

@@ -1,46 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use Test::More;
-use Data::Dumper;
-use JSON;
-
-my $tests = 0;
-
-unlink "tmp.json";
-pass("Running openssl based client simulations against smtp-relay.gmail.com:587"); $tests++;
-my $opensslout = `./testssl.sh --client-simulation --ssl-native -t smtp --jsonfile tmp.json --color 0 smtp-relay.gmail.com:587`;
-my $openssl = json('tmp.json');
-unlike($opensslout, qr/Running client simulations via sockets/, "Tests didn't run via sockets"); $tests++;
-
-pass("Running socket based client simulations against smtp-relay.gmail.com:587"); $tests++;
-unlink "tmp.json";
-my $socketout = `./testssl.sh --client-simulation -t smtp --jsonfile tmp.json --color 0 smtp-relay.gmail.com:587`;
-my $socket = json('tmp.json');
-like($socketout, qr/Running client simulations via sockets/, "Tests ran via sockets"); $tests++;
-
-
-#my $i = 0;
-#foreach my $o ( @$openssl ) {
-#	my $s = $$socket[$i];
-#	if ( $o->{id} =~ /^client_/ ) {
-#		pass("Comparing $o->{id}"); $tests++;
-#		cmp_ok($o->{id}, "eq", $s->{id}, "Id's match"); $tests++;
-#		cmp_ok($o->{severity}, "eq", $s->{severity}, "Severities match"); $tests++;
-#		cmp_ok($o->{finding}, "eq", $s->{finding}, "Findings match"); $tests++;
-#	}
-#	$i++;
-#}
-
-done_testing($tests);
-unlink "tmp.json";
-
-sub json($) {
-	my $file = shift;
-	$file = `cat $file`;
-	unlink $file;
-	return from_json($file);
-}
-
-# problem: 1-4 ok but of limited use: wy should we test whether runs really via sockets or openssl??
-# 5-n:     no sense, we know sockets and ssl are diffferent why should we have a unit test comparing those???

t/Readme.md

@@ -0,0 +1,7 @@
+### Naming scheme
+
+* 00-09:  Is reporting ok
+* 20-39:  Do scans work fine (client side)
+* 50-69:  Are the results what I expect? (server side)
+
+Please help to write Travis/CI tests! Documentation can be found [here](https://perldoc.perl.org/Test/More.html) and consulting the existing code (e.g. 20_baseline_ipv4_http.t)

testssl.sh

@@ -1891,6 +1891,14 @@ s_client_options() {
                fi
           fi
      fi
+
+     # OpenSSL's name for secp256r1 is prime256v1. So whenever we encounter this
+     # (e.g. client simulations) we replace it with the name which OpenSSL understands
+     # This shouldn't be needed. We have this here as a last resort
+     if [[ "$1" =~ " -curves " ]]; then
+          [[ "$1" =~ secp192r1 ]] && options="${options//secp192r1/prime192v1}"
+          [[ "$1" =~ secp256r1 ]] && options="${options//secp256r1/prime256v1}"
+     fi
      tm_out "$options"
 }
 
@@ -4496,6 +4504,11 @@ run_client_simulation() {
                               # "$OPENSSL s_client" will fail if the -curves option includes any unsupported curves.
                               supported_curves=""
                               for curve in $(colon_to_spaces "${curves[i]}"); do
+                                   # Attention! secp256r1 = prime256v1 and secp192r1 = prime192v1
+                                   # We need to map two curves here as otherwise handshakes will go wrong if "-curves" are supplied
+                                   # https://github.com/openssl/openssl/blob/master/apps/ecparam.c#L221 + ./ssl/t1_lib.c
+                                   [[ "$curve" =~ secp256r1 ]] && curve="${curve//secp256r1/prime256v1}"
+                                   [[ "$curve" =~ secp192r1 ]] && curve="${curve//secp192r1/prime192v1}"
                                    [[ "$OSSL_SUPPORTED_CURVES" =~ " $curve " ]] && supported_curves+=":$curve"
                               done
                               curves[i]=""
@@ -4541,6 +4554,15 @@ run_client_simulation() {
                          if [[ "$proto" == TLSv1.2 ]] && ( ! "$using_sockets" || [[ -z "${handshakebytes[i]}" ]] ); then
                               # OpenSSL reports TLS1.2 even if the connection is TLS1.1 or TLS1.0. Need to figure out which one it is...
                               for tls in ${tlsvers[i]}; do
+                                   # If the handshake data includes TLS 1.3 we need to remove it, otherwise the
+                                   # simulation will fail with # 'Oops: openssl s_client connect problem'
+                                   # before/after trying another protocol. We only print a warning it in debug mode
+                                   # as otherwise we would need e.g. handle the curves in a similar fashion -- not
+                                   # to speak about ciphers
+                                   if [[ $tls =~ 1_3 ]] && ! "$HAS_TLS13"; then
+                                        debugme pr_local_problem "TLS 1.3 not supported, "
+                                        continue
+                                   fi
                                    options="$(s_client_options "$tls -cipher ${ciphers[i]} -ciphersuites "\'${ciphersuites[i]}\'" ${curves[i]} $STARTTLS $BUGS $PROXY -connect $NODEIP:$PORT ${sni[i]}")"
                                    debugme echo "$OPENSSL s_client $options  </dev/null"
                                    $OPENSSL s_client $options  </dev/null >$TMPFILE 2>$ERRFILE
@@ -6773,6 +6795,12 @@ tls_time() {
      local jsonID="TLS_timestamp"
 
      pr_bold " TLS clock skew" ; out "$spaces"
+
+     if ( [[ "$STARTTLS_PROTOCOL" =~ ldap ]] || [[ "$STARTTLS_PROTOCOL" =~ irc ]] ); then
+          prln_local_problem "STARTTLS/$STARTTLS_PROTOCOL and --ssl-native collide here"
+          return 1
+     fi
+
      TLS_DIFFTIME_SET=true                                       # this is a switch whether we want to measure the remote TLS_TIME
      tls_sockets "01" "$TLS_CIPHER"                              # try first TLS 1.0 (most frequently used protocol)
      [[ -z "$TLS_TIME" ]] && tls_sockets "03" "$TLS12_CIPHER"    #           TLS 1.2
@@ -7079,7 +7107,10 @@ get_server_certificate() {
                extract_stapled_ocsp
                success=$?
           else
-               if [[ "$1" =~ "tls1_3_RSA" ]]; then
+               # For STARTTLS protcols not being implemented yet via sockets this is a bypass otherwise it won't be usable at all (e.g. LDAP)
+               if ( [[ "$STARTTLS" =~ ldap ]] || [[ "$STARTTLS" =~ irc ]] ); then
+                    return 1
+               elif [[ "$1" =~ "tls1_3_RSA" ]]; then
                     tls_sockets "04" "$TLS13_CIPHER" "all" "00,12,00,00, 00,05,00,05,01,00,00,00,00, 00,0d,00,10,00,0e,08,04,08,05,08,06,04,01,05,01,06,01,02,01"
                elif [[ "$1" =~ "tls1_3_ECDSA" ]]; then
                     tls_sockets "04" "$TLS13_CIPHER" "all" "00,12,00,00, 00,05,00,05,01,00,00,00,00, 00,0d,00,0a,00,08,04,03,05,03,06,03,02,03"
@@ -9105,7 +9136,7 @@ run_pfs() {
                     sigalg[nr_supported_ciphers]=""
                     ossl_supported[nr_supported_ciphers]="${TLS_CIPHER_OSSL_SUPPORTED[i]}"
                     hexcode[nr_supported_ciphers]="${hexc:2:2},${hexc:7:2}"
-                    if [[ "${hexc:2:2}" == "00" ]]; then
+                    if [[ "${hexc:2:2}" == 00 ]]; then
                          normalized_hexcode[nr_supported_ciphers]="x${hexc:7:2}"
                     else
                          normalized_hexcode[nr_supported_ciphers]="x${hexc:2:2}${hexc:7:2}"
@@ -9117,7 +9148,7 @@ run_pfs() {
      else
           while read -r hexc dash ciph[nr_supported_ciphers] sslvers kx[nr_supported_ciphers] auth enc[nr_supported_ciphers] mac export; do
                ciphers_found[nr_supported_ciphers]=false
-               if [[ "${hexc:2:2}" == "00" ]]; then
+               if [[ "${hexc:2:2}" == 00 ]]; then
                     normalized_hexcode[nr_supported_ciphers]="x${hexc:7:2}"
                else
                     normalized_hexcode[nr_supported_ciphers]="x${hexc:2:2}${hexc:7:2}"
@@ -9887,6 +9918,7 @@ starttls_mysql_dialog() {
 
 # arg1: fd for socket -- which we don't use as it is a hassle and it is not clear whether it works under every bash version
 # returns 6 if opening the socket caused a problem, 1 if STARTTLS handshake failed, 0: all ok
+#
 fd_socket() {
      local jabber=""
      local proyxline=""
@@ -9960,7 +9992,7 @@ fd_socket() {
                     fatal "FIXME: IRC+STARTTLS not yet supported" $ERR_NOSUPPORT
                     ;;
                ldap|ldaps) # LDAP, https://tools.ietf.org/html/rfc2830, https://tools.ietf.org/html/rfc4511
-                    fatal "FIXME: LDAP+STARTTLS over sockets not yet supported (try \"--ssl-native\")" $ERR_NOSUPPORT
+                    fatal "FIXME: LDAP+STARTTLS over sockets not supported yet (try \"--ssl-native\")" $ERR_NOSUPPORT
                     ;;
                acap|acaps) # ACAP = Application Configuration Access Protocol, see https://tools.ietf.org/html/rfc2595
                     fatal "ACAP Easteregg: not implemented -- probably never will" $ERR_NOSUPPORT
@@ -13291,12 +13323,10 @@ tls_sockets() {
 }
 
 
-####### vulnerabilities follow #######
+####### Vulnerabilities follow #######
-
+# General overview which browser "supports" which vulnerability:
-# general overview which browser "supports" which vulnerability:
 # http://en.wikipedia.org/wiki/Transport_Layer_Security-SSL#Web_browsers
 
-
 # mainly adapted from https://gist.github.com/takeshixx/10107280
 #
 run_heartbleed(){
@@ -13313,6 +13343,11 @@ run_heartbleed(){
      [[ $VULN_COUNT -le $VULN_THRESHLD ]] && outln && pr_headlineln " Testing for heartbleed vulnerability " && outln
      pr_bold " Heartbleed"; out " ($cve)                "
 
+     if ( [[ "$STARTTLS_PROTOCOL" =~ ldap ]] || [[ "$STARTTLS_PROTOCOL" =~ irc ]] ); then
+          prln_local_problem "STARTTLS/$STARTTLS_PROTOCOL and --ssl-native collide here"
+          return 1
+     fi
+
      [[ -z "$TLS_EXTENSIONS" ]] && determine_tls_extensions
      if [[ ! "${TLS_EXTENSIONS}" =~ heartbeat ]]; then
           pr_svrty_best "not vulnerable (OK)"
@@ -13367,8 +13402,8 @@ run_heartbleed(){
                tmln_out
           fi
 
-          if [[ $lines_returned -gt 1 ]] && [[ "${tls_hello_ascii:0:4}" == "1803" ]]; then
+          if [[ $lines_returned -gt 1 ]] && [[ "${tls_hello_ascii:0:4}" == 1803 ]]; then
-               if [[ "$STARTTLS_PROTOCOL" == "ftp" ]] || [[ "$STARTTLS_PROTOCOL" == "ftps" ]]; then
+               if [[ "$STARTTLS_PROTOCOL" =~ ftp ]]; then
                     # check possibility of weird vsftpd reply, see #426, despite "1803" seems very unlikely...
                     if grep -q '500 OOPS' "$SOCK_REPLY_FILE" ; then
                          append=", successful weeded out vsftpd false positive"
@@ -13417,6 +13452,11 @@ run_ccs_injection(){
      [[ $VULN_COUNT -le $VULN_THRESHLD ]] && outln && pr_headlineln " Testing for CCS injection vulnerability " && outln
      pr_bold " CCS"; out " ($cve)                       "
 
+     if ( [[ "$STARTTLS_PROTOCOL" =~ ldap ]] || [[ "$STARTTLS_PROTOCOL" =~ irc ]] ); then
+          prln_local_problem "STARTTLS/$STARTTLS_PROTOCOL and --ssl-native collide here"
+          return 1
+     fi
+
      if [[ 0 -eq $(has_server_protocol tls1) ]]; then
           tls_hexcode="x03, x01"
      elif [[ 0 -eq $(has_server_protocol tls1_1) ]]; then
@@ -13990,7 +14030,7 @@ run_crime() {
                [[ $sclient_success -eq 0 ]] && cp "$TEMPDIR/$NODEIP.parse_tls_serverhello.txt" $TMPFILE
           fi
      else
-          [[ "$OSSL_VER" == "0.9.8"* ]] && addcmd="-no_ssl2"
+          [[ "$OSSL_VER" == 0.9.8* ]] && addcmd="-no_ssl2"
           "$HAS_TLS13" && [[ -z "$OPTIMAL_PROTO" ]] && addcmd+=" -no_tls1_3"
           $OPENSSL s_client $(s_client_options "$OPTIMAL_PROTO $BUGS -comp $addcmd $STARTTLS -connect $NODEIP:$PORT $PROXY $SNI") </dev/null &>$TMPFILE
           sclient_connect_successful $? $TMPFILE
@@ -14857,7 +14897,17 @@ run_drown() {
           cert_fingerprint_sha2=${cert_fingerprint_sha2/SHA256 /}
      fi
 
+     if ( [[ "$STARTTLS_PROTOCOL" =~ ldap ]] || [[ "$STARTTLS_PROTOCOL" =~ irc ]] ); then
+          prln_local_problem "STARTTLS/$STARTTLS_PROTOCOL and --ssl-native collide here"
+          return 1
+     fi
+
+     if [[ $(has_server_protocol ssl2) -ne 1 ]]; then
           sslv2_sockets
+     else
+          [[ aaa == bbb ]]    # provoke retrurn code=1
+     fi
+
      case $? in
           7) # strange reply, couldn't convert the cipher spec length to a hex number
                pr_fixme "strange v2 reply "
@@ -14869,6 +14919,7 @@ run_drown() {
           3)   # vulnerable, [[ -n "$cert_fingerprint_sha2" ]] test is not needed as we should have RSA certificate here
                lines=$(count_lines "$(hexdump -C "$TEMPDIR/$NODEIP.sslv2_sockets.dd" 2>/dev/null)")
                debugme tm_out "  ($lines lines)  "
+               add_tls_offered ssl2 yes
                if [[ "$lines" -gt 1 ]]; then
                     nr_ciphers_detected=$((V2_HELLO_CIPHERSPEC_LENGTH / 3))
                     if [[ 0 -eq "$nr_ciphers_detected" ]]; then
@@ -15906,6 +15957,11 @@ run_robot() {
      [[ $VULN_COUNT -le $VULN_THRESHLD ]] && outln && pr_headlineln " Testing for Return of Bleichenbacher's Oracle Threat (ROBOT) vulnerability " && outln
      pr_bold " ROBOT                                     "
 
+     if ( [[ "$STARTTLS_PROTOCOL" =~ ldap ]] || [[ "$STARTTLS_PROTOCOL" =~ irc ]] ); then
+          prln_local_problem "STARTTLS/$STARTTLS_PROTOCOL and --ssl-native collide here"
+          return 1
+     fi
+
      if [[ ! "$HAS_PKUTIL" ]]; then
           prln_local_problem "Your $OPENSSL does not support the pkeyutl utility."
           fileout "$jsonID" "WARN" "$OPENSSL does not support the pkeyutl utility." "$cve" "$cwe"
@@ -16668,6 +16724,11 @@ HAS_NO_SSL2: $HAS_NO_SSL2
 HAS_SPDY: $HAS_SPDY
 HAS_ALPN: $HAS_ALPN
 HAS_FALLBACK_SCSV: $HAS_FALLBACK_SCSV
+HAS_COMP: $HAS_COMP
+HAS_NO_COMP: $HAS_NO_COMP
+HAS_CIPHERSUITES: $HAS_CIPHERSUITES
+HAS_PKEY: $HAS_PKEY
+HAS_PKUTIL: $HAS_PKUTIL
 HAS_PROXY: $HAS_PROXY
 HAS_XMPP: $HAS_XMPP
 HAS_POSTGRES: $HAS_POSTGRES
@@ -16886,7 +16947,7 @@ ip_fatal() {
      return 0
 }
 
-# This gneric function outputs an error onto the screen and handles logging.
+# This generic function outputs an error onto the screen and handles logging.
 # arg1: string to print / to write to file, arg2 (optional): additional hint to write
 #
 generic_nonfatal() {
@@ -17750,6 +17811,10 @@ determine_sizelimitbug() {
      local overflow_cipher1='C0,86'
      local overflow_cipher2='C0,88'
 
+     # For STARTTLS protcols not being implemented yet via sockets this is a bypass otherwise it  won't be usable at all (e.g. LDAP)
+     [[ "$STARTTLS" =~ ldap ]] && return 0
+     [[ "$STARTTLS" =~ irc ]] && return 0
+
      debugme echo -n "${FUNCNAME[0]} starting at # of ciphers (excl. 00FF): "
      debugme 'echo  "$test_ciphers" | tr ' ' '\n' | wc -l'
      # Only with TLS 1.2 offered at the server side it is possible to hit this bug, in practise. Thus
@@ -17947,6 +18012,7 @@ run_mx_all_ips() {
 # If run_mass_testing() is being used, then "$1" is "serial". If
 # run_mass_testing_parallel() is being used, then "$1" is "parallel XXXXXXXX"
 # where XXXXXXXX is the number of the test being run.
+#
 create_mass_testing_cmdline() {
      local testing_type="$1"
      local cmd test_number
@@ -18240,6 +18306,7 @@ run_mass_testing() {
 # appropriate, adds any JSON, CSV, and HTML output it has created to the
 # appropriate file. If the child process was stopped, then a message indicating
 # that is printed, but the incomplete results are not used.
+#
 get_next_message_testing_parallel_result() {
      draw_line "=" $((TERM_WIDTH / 2)); outln;
      outln "${PARALLEL_TESTING_CMDLINE[NEXT_PARALLEL_TEST_TO_FINISH]}"
@@ -18638,8 +18705,8 @@ parse_cmd_line() {
                     STARTTLS_PROTOCOL="$(parse_opt_equal_sign "$1" "$2")"
                     [[ $? -eq 0 ]] && shift
                     case $STARTTLS_PROTOCOL in
-                         ftp|smtp|lmtp|pop3|imap|xmpp|telnet|ldap|nntp|postgres|mysql) ;;
+                         ftp|smtp|lmtp|pop3|imap|xmpp|telnet|ldap|irc|nntp|postgres|mysql) ;;
-                         ftps|smtps|lmtps|pop3s|imaps|xmpps|telnets|ldaps|nntps) ;;
+                         ftps|smtps|lmtps|pop3s|imaps|xmpps|telnets|ldaps|ircs|nntps) ;;
                          *)   tmln_magenta "\nunrecognized STARTTLS protocol \"$1\", see help" 1>&2
                               help 1 ;;
                     esac
@@ -18672,7 +18739,7 @@ parse_cmd_line() {
                -c|--client-simulation)
                     do_client_simulation=true
                     ;;
-               -U|--vulnerable)
+               -U|--vulnerable|--vulnerabilities)
                     do_vulnerabilities=true
                     do_heartbleed="$OFFENSIVE"
                     do_ccs_injection="$OFFENSIVE"