Now reports policy errors in an easier to read format. (#63)

src/ssh_audit/policy.py

@@ -375,20 +375,23 @@ macs = %s
         '''Transforms an error struct to a flat string of error messages.'''
 
         error_list = []
+        spacer = ''
         for e in errors:
-            e_str = "%s did not match. " % e['mismatched_field']
+            e_str = "  * %s did not match.\n" % e['mismatched_field']
             if ('expected_optional' in e) and (e['expected_optional'] != ['']):
-                e_str += "Expected (required): %s; Expected (optional): %s" % (Policy._normalize_error_field(e['expected_required']), Policy._normalize_error_field(e['expected_optional']))
+                e_str += "    - Expected (required): %s\n    - Expected (optional): %s\n" % (Policy._normalize_error_field(e['expected_required']), Policy._normalize_error_field(e['expected_optional']))
+                spacer = '              '
             else:
-                e_str += "Expected: %s" % Policy._normalize_error_field(e['expected_required'])
+                e_str += "    - Expected: %s\n" % Policy._normalize_error_field(e['expected_required'])
-            e_str += "; Actual: %s" % Policy._normalize_error_field(e['actual'])
+                spacer = '   '
+            e_str += "    - Actual:%s%s\n" % (spacer, Policy._normalize_error_field(e['actual']))
             error_list.append(e_str)
 
         error_list.sort()  # To ensure repeatable results for testing.
 
         error_str = ''
         if len(error_list) > 0:
-            error_str = "  * %s" % '\n  * '.join(error_list)
+            error_str = "\n".join(error_list)
 
         return error_str
 
@@ -449,14 +452,14 @@ macs = %s
 
     @staticmethod
     def _normalize_error_field(field: List[str]) -> Any:
-        '''If field is an array with a string parsable as an integer, return that integer.  Otherwise, return the field unmodified.'''
+        '''If field is an array with a string parsable as an integer, return that integer.  Otherwise, return the field joined with commas.'''
         if len(field) == 1:
             try:
                 return int(field[0])
             except ValueError:
-                return field
+                return field[0]
         else:
-            return field
+            return ', '.join(field)
 
 
     def __str__(self) -> str:

test/docker/expected_results/openssh_5.6p1_custom_policy_test10.txt

@@ -3,5 +3,11 @@ Policy: Docker poliicy: test10 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * RSA CA key (ssh-rsa-cert-v01@openssh.com) sizes did not match. Expected: 4096; Actual: 1024
+  * RSA CA key (ssh-rsa-cert-v01@openssh.com) sizes did not match.
-  * RSA host key (ssh-rsa-cert-v01@openssh.com) sizes did not match. Expected: 4096; Actual: 3072
+    - Expected: 4096
+    - Actual:   1024
+
+  * RSA host key (ssh-rsa-cert-v01@openssh.com) sizes did not match.
+    - Expected: 4096
+    - Actual:   3072
+

test/docker/expected_results/openssh_5.6p1_custom_policy_test2.txt

@@ -3,4 +3,7 @@ Policy: Docker policy: test2 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * Key exchanges did not match. Expected: ['kex_alg1', 'kex_alg2']; Actual: ['diffie-hellman-group-exchange-sha256', 'diffie-hellman-group-exchange-sha1', 'diffie-hellman-group14-sha1', 'diffie-hellman-group1-sha1']
+  * Key exchanges did not match.
+    - Expected: kex_alg1, kex_alg2
+    - Actual:   diffie-hellman-group-exchange-sha256, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1
+

test/docker/expected_results/openssh_5.6p1_custom_policy_test3.txt

@@ -3,4 +3,7 @@ Policy: Docker policy: test3 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * Host keys did not match. Expected: ['ssh-rsa', 'ssh-dss', 'key_alg1']; Actual: ['ssh-rsa', 'ssh-dss']
+  * Host keys did not match.
+    - Expected: ssh-rsa, ssh-dss, key_alg1
+    - Actual:   ssh-rsa, ssh-dss
+

test/docker/expected_results/openssh_5.6p1_custom_policy_test4.txt

@@ -3,4 +3,7 @@ Policy: Docker policy: test4 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * Ciphers did not match. Expected: ['cipher_alg1', 'cipher_alg2']; Actual: ['aes128-ctr', 'aes192-ctr', 'aes256-ctr', 'arcfour256', 'arcfour128', 'aes128-cbc', '3des-cbc', 'blowfish-cbc', 'cast128-cbc', 'aes192-cbc', 'aes256-cbc', 'arcfour', 'rijndael-cbc@lysator.liu.se']
+  * Ciphers did not match.
+    - Expected: cipher_alg1, cipher_alg2
+    - Actual:   aes128-ctr, aes192-ctr, aes256-ctr, arcfour256, arcfour128, aes128-cbc, 3des-cbc, blowfish-cbc, cast128-cbc, aes192-cbc, aes256-cbc, arcfour, rijndael-cbc@lysator.liu.se
+

test/docker/expected_results/openssh_5.6p1_custom_policy_test5.txt

@@ -3,4 +3,7 @@ Policy: Docker policy: test5 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * MACs did not match. Expected: ['hmac-md5', 'hmac-sha1', 'umac-64@openssh.com', 'hmac-ripemd160', 'hmac-ripemd160@openssh.com', 'hmac_alg1', 'hmac-md5-96']; Actual: ['hmac-md5', 'hmac-sha1', 'umac-64@openssh.com', 'hmac-ripemd160', 'hmac-ripemd160@openssh.com', 'hmac-sha1-96', 'hmac-md5-96']
+  * MACs did not match.
+    - Expected: hmac-md5, hmac-sha1, umac-64@openssh.com, hmac-ripemd160, hmac-ripemd160@openssh.com, hmac_alg1, hmac-md5-96
+    - Actual:   hmac-md5, hmac-sha1, umac-64@openssh.com, hmac-ripemd160, hmac-ripemd160@openssh.com, hmac-sha1-96, hmac-md5-96
+

test/docker/expected_results/openssh_5.6p1_custom_policy_test8.txt

@@ -3,4 +3,7 @@ Policy: Docker poliicy: test8 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * RSA CA key (ssh-rsa-cert-v01@openssh.com) sizes did not match. Expected: 2048; Actual: 1024
+  * RSA CA key (ssh-rsa-cert-v01@openssh.com) sizes did not match.
+    - Expected: 2048
+    - Actual:   1024
+

test/docker/expected_results/openssh_5.6p1_custom_policy_test9.txt

@@ -3,4 +3,7 @@ Policy: Docker poliicy: test9 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * RSA host key (ssh-rsa-cert-v01@openssh.com) sizes did not match. Expected: 4096; Actual: 3072
+  * RSA host key (ssh-rsa-cert-v01@openssh.com) sizes did not match.
+    - Expected: 4096
+    - Actual:   3072
+

test/docker/expected_results/openssh_8.0p1_builtin_policy_test2.txt

@@ -3,4 +3,7 @@ Policy: Hardened OpenSSH Server v8.0 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * MACs did not match. Expected: ['hmac-sha2-256-etm@openssh.com', 'hmac-sha2-512-etm@openssh.com', 'umac-128-etm@openssh.com']; Actual: ['umac-64-etm@openssh.com', 'umac-128-etm@openssh.com', 'hmac-sha2-256-etm@openssh.com', 'hmac-sha2-512-etm@openssh.com', 'hmac-sha1-etm@openssh.com', 'umac-64@openssh.com', 'umac-128@openssh.com', 'hmac-sha2-256', 'hmac-sha2-512', 'hmac-sha1']
+  * MACs did not match.
+    - Expected: hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, umac-128-etm@openssh.com
+    - Actual:   umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+

test/docker/expected_results/openssh_8.0p1_custom_policy_test12.txt

@@ -3,6 +3,15 @@ Policy: Docker policy: test12 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * RSA host key (rsa-sha2-256) sizes did not match. Expected: 4096; Actual: 3072
+  * RSA host key (rsa-sha2-256) sizes did not match.
-  * RSA host key (rsa-sha2-512) sizes did not match. Expected: 4096; Actual: 3072
+    - Expected: 4096
-  * RSA host key (ssh-rsa) sizes did not match. Expected: 4096; Actual: 3072
+    - Actual:   3072
+
+  * RSA host key (rsa-sha2-512) sizes did not match.
+    - Expected: 4096
+    - Actual:   3072
+
+  * RSA host key (ssh-rsa) sizes did not match.
+    - Expected: 4096
+    - Actual:   3072
+

test/docker/expected_results/openssh_8.0p1_custom_policy_test14.txt

@@ -3,4 +3,7 @@ Policy: Docker policy: test14 (version 1)
 Result: ❌ Failed!
 
 Errors:
-  * Group exchange (diffie-hellman-group-exchange-sha256) modulus sizes did not match. Expected: 4096; Actual: 2048
+  * Group exchange (diffie-hellman-group-exchange-sha256) modulus sizes did not match.
+    - Expected: 4096
+    - Actual:   2048
+