Added JSON output tests to docker testing suite.

docker_test.sh

@@ -385,24 +385,32 @@ function run_test {
     options=$4
 
     server_exec=
-    test_result=
+    test_result_stdout=
-    expected_result=
+    test_result_json=
+    expected_result_stdout=
+    expected_result_json=
     test_name=
     if [[ $server_type == 'OpenSSH' ]]; then
 	server_exec="/openssh/sshd-${version} -D -f /etc/ssh/sshd_config-${version}_${test_number}"
-	test_result="${TEST_RESULT_DIR}/openssh_${version}_${test_number}.txt"
+	test_result_stdout="${TEST_RESULT_DIR}/openssh_${version}_${test_number}.txt"
-	expected_result="test/docker/expected_results/openssh_${version}_${test_number}.txt"
+	test_result_json="${TEST_RESULT_DIR}/openssh_${version}_${test_number}.json"
+	expected_result_stdout="test/docker/expected_results/openssh_${version}_${test_number}.txt"
+	expected_result_json="test/docker/expected_results/openssh_${version}_${test_number}.json"
 	test_name="OpenSSH ${version} ${test_number}"
 	options=
     elif [[ $server_type == 'Dropbear' ]]; then
 	server_exec="/dropbear/dropbear-${version} -F ${options}"
-	test_result="${TEST_RESULT_DIR}/dropbear_${version}_${test_number}.txt"
+	test_result_stdout="${TEST_RESULT_DIR}/dropbear_${version}_${test_number}.txt"
-	expected_result="test/docker/expected_results/dropbear_${version}_${test_number}.txt"
+	test_result_json="${TEST_RESULT_DIR}/dropbear_${version}_${test_number}.json"
+	expected_result_stdout="test/docker/expected_results/dropbear_${version}_${test_number}.txt"
+	expected_result_json="test/docker/expected_results/dropbear_${version}_${test_number}.json"
 	test_name="Dropbear ${version} ${test_number}"
     elif [[ $server_type == 'TinySSH' ]]; then
 	server_exec="/usr/bin/tcpserver -HRDl0 0.0.0.0 22 /tinysshd/tinyssh-20190101 -v /etc/tinyssh/"
-	test_result="${TEST_RESULT_DIR}/tinyssh_${version}_${test_number}.txt"
+	test_result_stdout="${TEST_RESULT_DIR}/tinyssh_${version}_${test_number}.txt"
-	expected_result="test/docker/expected_results/tinyssh_${version}_${test_number}.txt"
+	test_result_json="${TEST_RESULT_DIR}/tinyssh_${version}_${test_number}.json"
+	expected_result_stdout="test/docker/expected_results/tinyssh_${version}_${test_number}.txt"
+	expected_result_json="test/docker/expected_results/tinyssh_${version}_${test_number}.json"
 	test_name="TinySSH ${version} ${test_number}"
     fi
 
@@ -412,7 +420,14 @@ function run_test {
 	exit 1
     fi
 
-    ./ssh-audit.py localhost:2222 > $test_result
+    ./ssh-audit.py localhost:2222 > $test_result_stdout
+    if [[ $? != 0 ]]; then
+	echo -e "${REDB}Failed to run ssh-audit.py! (exit code: $?)${CLR}"
+	docker container stop $cid > /dev/null
+	exit 1
+    fi
+
+    ./ssh-audit.py -j localhost:2222 > $test_result_json
     if [[ $? != 0 ]]; then
 	echo -e "${REDB}Failed to run ssh-audit.py! (exit code: $?)${CLR}"
 	docker container stop $cid > /dev/null
@@ -429,19 +444,25 @@ function run_test {
     # we need to filter out the banner part of the output so we get stable, repeatable
     # results.
     if [[ $server_type == 'TinySSH' ]]; then
-	grep -v "(gen) banner: " ${test_result} > "${test_result}.tmp"
+	grep -v "(gen) banner: " ${test_result_stdout} > "${test_result_stdout}.tmp"
-	mv "${test_result}.tmp" ${test_result}
+	mv "${test_result_stdout}.tmp" ${test_result_stdout}
+	cat "${test_result_json}" | perl -pe 's/"comments": ".*?"/"comments": ""/' | perl -pe 's/"raw": ".+?"/"raw": ""/' > "${test_result_json}.tmp"
+	mv "${test_result_json}.tmp" ${test_result_json}
     fi
 
-    diff=`diff -u ${expected_result} ${test_result}`
+    diff=`diff -u ${expected_result_stdout} ${test_result_stdout}`
-    if [[ $? == 0 ]]; then
+    if [[ $? != 0 ]]; then
-	echo -e "${test_name} ${GREEN}passed${CLR}."
-    else
 	echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
 	exit 1
     fi
-}
 
+    diff=`diff -u ${expected_result_json} ${test_result_json}`
+    if [[ $? != 0 ]]; then
+	echo -e "${test_name} ${REDB}FAILED${CLR}.\n\n${diff}\n"
+	exit 1
+    fi
+    echo -e "${test_name} ${GREEN}passed${CLR}."
+}
 
 
 # First check if docker is functional.

ssh-audit.py

@@ -3184,16 +3184,20 @@ def build_struct(banner, kex=None, pkm=None):
 		res['mac'] = kex.server.mac
 		res['fingerprints'] = []
 		host_keys = kex.host_keys()
-		for host_key_type in host_keys:
+
+		# Normalize all RSA key types to 'ssh-rsa'.  Otherwise, due to Python's order-indifference dictionary types, we would iterate key types in unpredictable orders, which interferes with the docker testing framework (i.e.: tests would fail because elements are reported out of order, even though the output is semantically the same).
+		for host_key_type in host_keys.keys():
+			if host_key_type in SSH2.HostKeyTest.RSA_FAMILY:
+				val = host_keys[host_key_type]
+				del(host_keys[host_key_type])
+				host_keys['ssh-rsa'] = val
+
+		for host_key_type in sorted(host_keys):
 			if host_keys[host_key_type] is None:
 				continue
 
 			fp = SSH.Fingerprint(host_keys[host_key_type])
 
-			# Workaround for Python's order-indifference in dicts.  We might get a random RSA type (ssh-rsa, rsa-sha2-256, or rsa-sha2-512), so running the tool against the same server three times may give three different host key types here.  So if we have any RSA type, we will simply hard-code it to 'ssh-rsa'.
-			if host_key_type in SSH2.HostKeyTest.RSA_FAMILY:
-				host_key_type = 'ssh-rsa'
-
 			# Skip over certificate host types (or we would return invalid fingerprints).
 			if '-cert-' in host_key_type:
 				continue
@@ -3265,7 +3269,7 @@ def audit(aconf, sshv=None):
 	if sshv == 1:
 		pkm = SSH1.PublicKeyMessage.parse(payload)
 		if aconf.json:
-			print(json.dumps(build_struct(banner, pkm=pkm)))
+			print(json.dumps(build_struct(banner, pkm=pkm), sort_keys=True))
 		else:
 			output(banner, header, pkm=pkm)
 	elif sshv == 2:
@@ -3274,7 +3278,7 @@ def audit(aconf, sshv=None):
 			SSH2.HostKeyTest.run(s, kex)
 			SSH2.GEXTest.run(s, kex)
 		if aconf.json:
-			print(json.dumps(build_struct(banner, kex=kex)))
+			print(json.dumps(build_struct(banner, kex=kex), sort_keys=True))
 		else:
 			output(banner, header, client_audit=aconf.client_audit, kex=kex)
 

test/docker/expected_results/dropbear_2019.78_test1.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-dropbear_2019.78", "software": "dropbear_2019.78"}, "compression": ["zlib@openssh.com", "none"], "enc": ["aes128-ctr", "aes256-ctr", "aes128-cbc", "aes256-cbc", "3des-ctr", "3des-cbc"], "fingerprints": [{"fp": "SHA256:CDfAU12pjQS7/91kg7gYacza0U/6PDbE04Ic3IpYxkM", "type": "ssh-rsa"}], "kex": [{"algorithm": "curve25519-sha256"}, {"algorithm": "curve25519-sha256@libssh.org"}, {"algorithm": "ecdh-sha2-nistp521"}, {"algorithm": "ecdh-sha2-nistp384"}, {"algorithm": "ecdh-sha2-nistp256"}, {"algorithm": "diffie-hellman-group14-sha256"}, {"algorithm": "diffie-hellman-group14-sha1"}, {"algorithm": "kexguess2@matt.ucc.asn.au"}], "key": [{"algorithm": "ecdsa-sha2-nistp256"}, {"algorithm": "ssh-rsa", "keysize": 1024}, {"algorithm": "ssh-dss"}], "mac": ["hmac-sha1-96", "hmac-sha1", "hmac-sha2-256"]}

test/docker/expected_results/openssh_4.0p1_test1.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [1, 99], "raw": "SSH-1.99-OpenSSH_4.0", "software": "OpenSSH_4.0"}, "compression": ["none", "zlib"], "enc": ["aes128-cbc", "3des-cbc", "blowfish-cbc", "cast128-cbc", "arcfour", "aes192-cbc", "aes256-cbc", "rijndael-cbc@lysator.liu.se", "aes128-ctr", "aes192-ctr", "aes256-ctr"], "fingerprints": [{"fp": "SHA256:YZ457EBcJTSxRKI3yXRgtAj3PBf5B9/F36b1SVooml4", "type": "ssh-rsa"}], "kex": [{"algorithm": "diffie-hellman-group-exchange-sha1", "keysize": 1024}, {"algorithm": "diffie-hellman-group14-sha1"}, {"algorithm": "diffie-hellman-group1-sha1"}], "key": [{"algorithm": "ssh-rsa", "keysize": 1024}, {"algorithm": "ssh-dss"}], "mac": ["hmac-md5", "hmac-sha1", "hmac-ripemd160", "hmac-ripemd160@openssh.com", "hmac-sha1-96", "hmac-md5-96"]}

test/docker/expected_results/openssh_5.6p1_test1.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-OpenSSH_5.6", "software": "OpenSSH_5.6"}, "compression": ["none", "zlib@openssh.com"], "enc": ["aes128-ctr", "aes192-ctr", "aes256-ctr", "arcfour256", "arcfour128", "aes128-cbc", "3des-cbc", "blowfish-cbc", "cast128-cbc", "aes192-cbc", "aes256-cbc", "arcfour", "rijndael-cbc@lysator.liu.se"], "fingerprints": [{"fp": "SHA256:YZ457EBcJTSxRKI3yXRgtAj3PBf5B9/F36b1SVooml4", "type": "ssh-rsa"}], "kex": [{"algorithm": "diffie-hellman-group-exchange-sha256", "keysize": 1024}, {"algorithm": "diffie-hellman-group-exchange-sha1", "keysize": 1024}, {"algorithm": "diffie-hellman-group14-sha1"}, {"algorithm": "diffie-hellman-group1-sha1"}], "key": [{"algorithm": "ssh-rsa", "keysize": 1024}, {"algorithm": "ssh-dss"}], "mac": ["hmac-md5", "hmac-sha1", "umac-64@openssh.com", "hmac-ripemd160", "hmac-ripemd160@openssh.com", "hmac-sha1-96", "hmac-md5-96"]}

test/docker/expected_results/openssh_5.6p1_test2.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-OpenSSH_5.6", "software": "OpenSSH_5.6"}, "compression": ["none", "zlib@openssh.com"], "enc": ["aes128-ctr", "aes192-ctr", "aes256-ctr", "arcfour256", "arcfour128", "aes128-cbc", "3des-cbc", "blowfish-cbc", "cast128-cbc", "aes192-cbc", "aes256-cbc", "arcfour", "rijndael-cbc@lysator.liu.se"], "fingerprints": [{"fp": "SHA256:YZ457EBcJTSxRKI3yXRgtAj3PBf5B9/F36b1SVooml4", "type": "ssh-rsa"}], "kex": [{"algorithm": "diffie-hellman-group-exchange-sha256", "keysize": 1024}, {"algorithm": "diffie-hellman-group-exchange-sha1", "keysize": 1024}, {"algorithm": "diffie-hellman-group14-sha1"}, {"algorithm": "diffie-hellman-group1-sha1"}], "key": [{"algorithm": "ssh-rsa", "keysize": 1024}, {"algorithm": "ssh-rsa-cert-v01@openssh.com", "casize": 1024, "keysize": 1024}], "mac": ["hmac-md5", "hmac-sha1", "umac-64@openssh.com", "hmac-ripemd160", "hmac-ripemd160@openssh.com", "hmac-sha1-96", "hmac-md5-96"]}

test/docker/expected_results/openssh_5.6p1_test3.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-OpenSSH_5.6", "software": "OpenSSH_5.6"}, "compression": ["none", "zlib@openssh.com"], "enc": ["aes128-ctr", "aes192-ctr", "aes256-ctr", "arcfour256", "arcfour128", "aes128-cbc", "3des-cbc", "blowfish-cbc", "cast128-cbc", "aes192-cbc", "aes256-cbc", "arcfour", "rijndael-cbc@lysator.liu.se"], "fingerprints": [{"fp": "SHA256:YZ457EBcJTSxRKI3yXRgtAj3PBf5B9/F36b1SVooml4", "type": "ssh-rsa"}], "kex": [{"algorithm": "diffie-hellman-group-exchange-sha256", "keysize": 1024}, {"algorithm": "diffie-hellman-group-exchange-sha1", "keysize": 1024}, {"algorithm": "diffie-hellman-group14-sha1"}, {"algorithm": "diffie-hellman-group1-sha1"}], "key": [{"algorithm": "ssh-rsa", "keysize": 1024}, {"algorithm": "ssh-rsa-cert-v01@openssh.com", "casize": 3072, "keysize": 1024}], "mac": ["hmac-md5", "hmac-sha1", "umac-64@openssh.com", "hmac-ripemd160", "hmac-ripemd160@openssh.com", "hmac-sha1-96", "hmac-md5-96"]}

test/docker/expected_results/openssh_5.6p1_test4.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-OpenSSH_5.6", "software": "OpenSSH_5.6"}, "compression": ["none", "zlib@openssh.com"], "enc": ["aes128-ctr", "aes192-ctr", "aes256-ctr", "arcfour256", "arcfour128", "aes128-cbc", "3des-cbc", "blowfish-cbc", "cast128-cbc", "aes192-cbc", "aes256-cbc", "arcfour", "rijndael-cbc@lysator.liu.se"], "fingerprints": [{"fp": "SHA256:nsWtdJ9Z67Vrf7OsUzQov7esXhsWAfVppArGh25u244", "type": "ssh-rsa"}], "kex": [{"algorithm": "diffie-hellman-group-exchange-sha256", "keysize": 1024}, {"algorithm": "diffie-hellman-group-exchange-sha1", "keysize": 1024}, {"algorithm": "diffie-hellman-group14-sha1"}, {"algorithm": "diffie-hellman-group1-sha1"}], "key": [{"algorithm": "ssh-rsa", "keysize": 3072}, {"algorithm": "ssh-rsa-cert-v01@openssh.com", "casize": 1024, "keysize": 3072}], "mac": ["hmac-md5", "hmac-sha1", "umac-64@openssh.com", "hmac-ripemd160", "hmac-ripemd160@openssh.com", "hmac-sha1-96", "hmac-md5-96"]}

test/docker/expected_results/openssh_5.6p1_test5.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-OpenSSH_5.6", "software": "OpenSSH_5.6"}, "compression": ["none", "zlib@openssh.com"], "enc": ["aes128-ctr", "aes192-ctr", "aes256-ctr", "arcfour256", "arcfour128", "aes128-cbc", "3des-cbc", "blowfish-cbc", "cast128-cbc", "aes192-cbc", "aes256-cbc", "arcfour", "rijndael-cbc@lysator.liu.se"], "fingerprints": [{"fp": "SHA256:nsWtdJ9Z67Vrf7OsUzQov7esXhsWAfVppArGh25u244", "type": "ssh-rsa"}], "kex": [{"algorithm": "diffie-hellman-group-exchange-sha256", "keysize": 1024}, {"algorithm": "diffie-hellman-group-exchange-sha1", "keysize": 1024}, {"algorithm": "diffie-hellman-group14-sha1"}, {"algorithm": "diffie-hellman-group1-sha1"}], "key": [{"algorithm": "ssh-rsa", "keysize": 3072}, {"algorithm": "ssh-rsa-cert-v01@openssh.com", "casize": 3072, "keysize": 3072}], "mac": ["hmac-md5", "hmac-sha1", "umac-64@openssh.com", "hmac-ripemd160", "hmac-ripemd160@openssh.com", "hmac-sha1-96", "hmac-md5-96"]}

test/docker/expected_results/openssh_8.0p1_test1.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-OpenSSH_8.0", "software": "OpenSSH_8.0"}, "compression": ["none", "zlib@openssh.com"], "enc": ["chacha20-poly1305@openssh.com", "aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "aes256-gcm@openssh.com"], "fingerprints": [{"fp": "SHA256:UrnXIVH+7dlw8UqYocl48yUEcKrthGDQG2CPCgp7MxU", "type": "ssh-ed25519"}, {"fp": "SHA256:nsWtdJ9Z67Vrf7OsUzQov7esXhsWAfVppArGh25u244", "type": "ssh-rsa"}], "kex": [{"algorithm": "curve25519-sha256"}, {"algorithm": "curve25519-sha256@libssh.org"}, {"algorithm": "ecdh-sha2-nistp256"}, {"algorithm": "ecdh-sha2-nistp384"}, {"algorithm": "ecdh-sha2-nistp521"}, {"algorithm": "diffie-hellman-group-exchange-sha256", "keysize": 2048}, {"algorithm": "diffie-hellman-group16-sha512"}, {"algorithm": "diffie-hellman-group18-sha512"}, {"algorithm": "diffie-hellman-group14-sha256"}, {"algorithm": "diffie-hellman-group14-sha1"}], "key": [{"algorithm": "rsa-sha2-512", "keysize": 3072}, {"algorithm": "rsa-sha2-256", "keysize": 3072}, {"algorithm": "ssh-rsa", "keysize": 3072}, {"algorithm": "ecdsa-sha2-nistp256"}, {"algorithm": "ssh-ed25519"}], "mac": ["umac-64-etm@openssh.com", "umac-128-etm@openssh.com", "hmac-sha2-256-etm@openssh.com", "hmac-sha2-512-etm@openssh.com", "hmac-sha1-etm@openssh.com", "umac-64@openssh.com", "umac-128@openssh.com", "hmac-sha2-256", "hmac-sha2-512", "hmac-sha1"]}

test/docker/expected_results/openssh_8.0p1_test2.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-OpenSSH_8.0", "software": "OpenSSH_8.0"}, "compression": ["none", "zlib@openssh.com"], "enc": ["chacha20-poly1305@openssh.com", "aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "aes256-gcm@openssh.com"], "fingerprints": [{"fp": "SHA256:UrnXIVH+7dlw8UqYocl48yUEcKrthGDQG2CPCgp7MxU", "type": "ssh-ed25519"}], "kex": [{"algorithm": "curve25519-sha256"}, {"algorithm": "curve25519-sha256@libssh.org"}, {"algorithm": "ecdh-sha2-nistp256"}, {"algorithm": "ecdh-sha2-nistp384"}, {"algorithm": "ecdh-sha2-nistp521"}, {"algorithm": "diffie-hellman-group-exchange-sha256", "keysize": 2048}, {"algorithm": "diffie-hellman-group16-sha512"}, {"algorithm": "diffie-hellman-group18-sha512"}, {"algorithm": "diffie-hellman-group14-sha256"}, {"algorithm": "diffie-hellman-group14-sha1"}], "key": [{"algorithm": "ssh-ed25519"}, {"algorithm": "ssh-ed25519-cert-v01@openssh.com"}], "mac": ["umac-64-etm@openssh.com", "umac-128-etm@openssh.com", "hmac-sha2-256-etm@openssh.com", "hmac-sha2-512-etm@openssh.com", "hmac-sha1-etm@openssh.com", "umac-64@openssh.com", "umac-128@openssh.com", "hmac-sha2-256", "hmac-sha2-512", "hmac-sha1"]}

test/docker/expected_results/openssh_8.0p1_test3.json

@@ -0,0 +1 @@
+{"banner": {"comments": null, "protocol": [2, 0], "raw": "SSH-2.0-OpenSSH_8.0", "software": "OpenSSH_8.0"}, "compression": ["none", "zlib@openssh.com"], "enc": ["chacha20-poly1305@openssh.com", "aes256-gcm@openssh.com", "aes128-gcm@openssh.com", "aes256-ctr", "aes192-ctr", "aes128-ctr"], "fingerprints": [{"fp": "SHA256:UrnXIVH+7dlw8UqYocl48yUEcKrthGDQG2CPCgp7MxU", "type": "ssh-ed25519"}], "kex": [{"algorithm": "curve25519-sha256"}, {"algorithm": "curve25519-sha256@libssh.org"}, {"algorithm": "diffie-hellman-group-exchange-sha256", "keysize": 2048}], "key": [{"algorithm": "ssh-ed25519"}], "mac": ["hmac-sha2-256-etm@openssh.com", "hmac-sha2-512-etm@openssh.com", "umac-128-etm@openssh.com"]}

test/docker/expected_results/tinyssh_20190101_test1.json

@@ -0,0 +1 @@
+{"banner": {"comments": "", "protocol": [2, 0], "raw": "", "software": "tinyssh_noversion"}, "compression": ["none"], "enc": ["chacha20-poly1305@openssh.com"], "fingerprints": [{"fp": "SHA256:89ocln1x7KNqnMgWffGoYtD70ksJ4FrH7BMJHa7SrwU", "type": "ssh-ed25519"}], "kex": [{"algorithm": "curve25519-sha256"}, {"algorithm": "curve25519-sha256@libssh.org"}, {"algorithm": "sntrup4591761x25519-sha512@tinyssh.org"}], "key": [{"algorithm": "ssh-ed25519"}], "mac": ["hmac-sha2-256"]}