diff --git a/correlate-rules.py b/correlate-rules.py index ceb5b58..76578c8 100644 --- a/correlate-rules.py +++ b/correlate-rules.py @@ -11,10 +11,14 @@ import glob import base64 rules = {} +files_and_their_rules = {} scanned = set() +FILES_PREFIX='analysis-' + def walk(path): global rules + global files_and_their_rules global scanned print(f'Walking {path}...') @@ -23,7 +27,14 @@ def walk(path): if not file.lower().endswith('.txt'): continue - if file in scanned: continue + if file in scanned: + continue + + base = os.path.basename(file) + if len(FILES_PREFIX) > 0: + if not base.lower().startswith(FILES_PREFIX.lower()): + continue + scanned.add(file) data = '' @@ -33,6 +44,11 @@ def walk(path): for m in re.finditer(r'(\(\d{4,}\))', data, re.I): rule = m.group(1) + if file not in files_and_their_rules.keys(): + files_and_their_rules[file] = set() + + files_and_their_rules[file].add(rule) + if rule in rules.keys(): if file not in rules[rule]['files']: rules[rule]['count'] += 1 @@ -57,7 +73,6 @@ def main(argv): print(f'[.] Found {len(rules)} unique rules.:') - candidates = [] for k, v in rules.items(): if v['count'] > 1: print(f'\n\t- {k: <15}: occurences: {v["count"]} - files: {len(v["files"])}') @@ -66,5 +81,38 @@ def main(argv): for f in v['files']: print('\t\t- ' + str(f)) + + output = ' # | file1 | file2 |\n' + output+= '----+----------------------------------------------------+----------------------------------------------------+\n' + + checked = set() + for k, v in files_and_their_rules.items(): + for k1, v1 in files_and_their_rules.items(): + if k == k1: + continue + + n = max(len(v.difference(v1)), len(v1.difference(v))) + if n <= 3 and n > 0: + if k not in checked and k1 not in checked: + output += f' {n: <2} | {k[-50:]: <50} | {k1[-50:]: <50} |\n' + checked.add(k) + checked.add(k1) + + output+= '----+----------------------------------------------------+----------------------------------------------------+\n' + + print('\nCross-File rules differences:\n') + print(output) + + print('\n\nFiles and rules matched:\n') + + num = 0 + s = {k: v for k, v in sorted(files_and_their_rules.items(), key=lambda item: len(item[1]))} + + for k, v in s.items(): + num += 1 + print(f'{num: <3}. Rules: {len(v): <2}, File: {k}') + + print() + if __name__ == '__main__': main(sys.argv) \ No newline at end of file diff --git a/decode-spam-headers.py b/decode-spam-headers.py index faefa4e..e9c460b 100644 --- a/decode-spam-headers.py +++ b/decode-spam-headers.py @@ -77,6 +77,8 @@ # - X-microsoft-antispam-untrusted # - X-sophos-senderhistory # - X-sophos-rescan +# - X-MS-Exchange-CrossTenant-Id +# - X-OriginatorOrg # # Usage: # ./decode-spam-headers [options] @@ -91,6 +93,7 @@ # Requirements: # - packaging # - dnspython +# - requests # # Mariusz Banach / mgeeky, '21 # @@ -121,6 +124,16 @@ except ImportError: ''') sys.exit(1) +try: + import requests +except ImportError: + print(''' +[!] You need to install requests: + # pip3 install requests +''') + sys.exit(1) + + try: import dns.resolver @@ -360,7 +373,7 @@ class SMTPHeadersAnalysis: 'yandex', 'yandexbot', 'zillya', 'zonealarm', 'zscaler', '-sea-', 'perlmx', 'trustwave', 'mailmarshal', 'tmase', 'startscan', 'fe-etp', 'jemd', 'suspicious', 'grey', 'infected', 'unscannable', 'dlp-', 'sanitize', 'mailscan', 'barracuda', 'clearswift', 'messagelabs', 'msw-jemd', 'fe-etp', 'symc-ess', - 'starscan', 'mailcontrol', + 'starscan', 'mailcontrol' ) Interesting_Headers = ( @@ -372,7 +385,35 @@ class SMTPHeadersAnalysis: 'X-ICPINFO', 'x-locaweb-id', 'X-MC-User', 'mailersend', 'MailiGen', 'Mandrill', 'MarketoID', 'X-Messagebus-Info', 'Mixmax', 'X-PM-Message-Id', 'postmark', 'X-rext', 'responsys', 'X-SFDC-User', 'salesforce', 'x-sg-', 'x-sendgrid-', 'silverpop', '.mkt', 'X-SMTPCOM-Tracking-Number', 'X-vrfbldomain', 'verticalresponse', - 'yesmail', + 'yesmail', 'logon', 'safelink', 'safeattach', + ) + + Security_Appliances_And_Their_Headers = \ + ( + ('Exchange Online Protection' , 'X-EOP'), + ('MS Defender For Office365' , '-Safelinks'), + ('Proofpoint Email Protection' , 'X-Proofpoint'), + ('Trend Micro Anti-Spam' , 'X-TMASE-'), + ('Trend Micro Anti-Spam' , 'X-TM-AS-'), + ('Trend Micro InterScan Messaging Security' , 'X-IMSS-'), + ('Barracuda Email Security' , 'X-Barracuda-'), + ('Mimecast' , 'X-Mimecast-'), + ('FireEye Email Security Solution' , 'X-FireEye'), + ('FireEye Email Security Solution' , 'X-FE-'), + ('Cisco IronPort' , 'X-IronPort-'), + ('Cisco IronPort / Email Security Appliance (ESA)' , 'X-Policy'), + ('Cisco IronPort / Email Security Appliance (ESA)' , 'X-SBRS'), + ('Sophos Email Appliance (PureMessage)' , 'X-SEA-'), + ('Exchange Server 2016 Anti-Spam' , 'SpamDiagnostic'), + ('SpamAssassin' , 'X-Spam-'), + ('SpamAssassin' , 'X-IP-Spam-'), + ('OVH Anti-Spam' , 'X-VR-'), + ('OVH Anti-Spam' , 'X-Ovh-'), + ('MS Defender Advanced Threat Protection' , 'X-MS.+-Atp'), + ('MS Defender Advanced Threat Protection - Safe Links' , '-ATPSafeLinks'), + ('Cisco Advanced Malware Protection (AMP)' , 'X-Amp-'), + ('MS ForeFront Anti-Spam' , 'X-Microsoft-Antispam'), + ('MS ForeFront Anti-Spam' , 'X-Forefront-Antispam'), ) Headers_Known_For_Breaking_Line = ( @@ -694,7 +735,7 @@ class SMTPHeadersAnalysis: # Message contained tag with URL containing GET parameter with value of another URL: ex. href="https://foo.bar/file?aaa=https://baz.xyz/"', + '45080400002' : 'Something about tag\'s URL. Possibly it contained GET parameter with value of another URL: ex. href="https://foo.bar/file?aaa=https://baz.xyz/"', # Message contained with href pointing to a file with dangerous extension, such as file.exe '460985005' : 'Mail body contained HTML tag with href URL pointing to a file with dangerous extension (such as .exe)', @@ -709,6 +750,29 @@ class SMTPHeadersAnalysis: # '121216002' : 'First Hop MTA SMTP Server used as a SMTP Relay. It\'s known to originate e-mails, but here it acted as a Relay. Or maybe due to use of "with ESMTPSA" instead of ESMTPS?', + # Triggered on message with added to HTML: https://www.reddit.com/ + '966005' : 'Mail body contained link tag with potentially masqueraded URL: https://example.com', + + # + # Message1: GoPhish EC2 -> another EC2 with socat to smtp.gmail.com:587 (authenticated) -> Target + # Message2: GoPhish EC2 -> Gsuite -> Target + # + # Subject, mail body were exactly the same. + # + # Below two rules were added to the second message. My understanding is that they're somehow referring + # to the reputation of the first-hop server, maybe reverse-DNS resolution. + # + '5002400100002' : "(GUESSING) Somehow related to First Hop server reputation, it's reverse-PTR resolution or domain impersonation", + '58800400005' : "(GUESSING) Somehow related to First Hop server reputation, it's reverse-PTR resolution or domain impersonation", + + '19625305002' : '(GUESSING) Something to do with the HTML code and used tags/structures', + '43540500002' : '(GUESSING) Something to do with the HTML code and used tags/structures', + + '460985005' : '(GUESSING) Something to do with either more-complex HTML code or with the tag and its URL.', + + # Triggered on an empty text message, subject "test" - that was marked with "Domain Impersonation", however + # ForeFront Anti-Spam headers did not support that Domain Impersonation. Weird. + '22186003' : '(GUESSING) Something to do with either Text message (non-HTML) or probable Domain Impersonation' } ForeFront_Spam_Confidence_Levels = { @@ -733,9 +797,9 @@ class SMTPHeadersAnalysis: ForeFront_Bulk_Confidence_Levels = { 0 : logger.colored('The message isn\'t from a bulk sender.', 'green'), - 1 : logger.colored('The message is from a bulk sender that generates few complaints.', 'magenta'), - 2 : logger.colored('The message is from a bulk sender that generates few complaints.', 'magenta'), - 3 : logger.colored('The message is from a bulk sender that generates few complaints.', 'magenta'), + 1 : logger.colored('The message is from a bulk sender that generates few complaints.', 'yellow'), + 2 : logger.colored('The message is from a bulk sender that generates few complaints.', 'yellow'), + 3 : logger.colored('The message is from a bulk sender that generates few complaints.', 'yellow'), 4 : logger.colored('The message is from a bulk sender that generates a mixed number of complaints.', 'red'), 5 : logger.colored('The message is from a bulk sender that generates a mixed number of complaints.', 'red'), 6 : logger.colored('The message is from a bulk sender that generates a mixed number of complaints.', 'red'), @@ -791,6 +855,7 @@ class SMTPHeadersAnalysis: { 'I' : logger.colored('Inbox directory', 'green'), 'J' : logger.colored('JUNK directory', 'red'), + 'C' : logger.colored('Custom directory', 'yellow'), } ), @@ -1128,13 +1193,16 @@ class SMTPHeadersAnalysis: ('66', 'X-Proofpoint-Spam-Details', self.testXProofpointSpamDetails), ('67', 'X-Proofpoint-Virus-Version', self.testXProofpointVirusVersion), ('68', 'SPFCheck', self.testSPFCheck), - ('69', 'X-Barracuda-Spam-Score', self.testXBarracudaSpamScore), ('70', 'X-Barracuda-Spam-Status', self.testXBarracudaSpamStatus), ('71', 'X-Barracuda-Spam-Report', self.testXBarracudaSpamReport), ('72', 'X-Barracuda-Bayes', self.testXBarracudaBayes), ('73', 'X-Barracuda-Start-Time', self.testXBarracudaStartTime), + ('83', 'Office365 Tenant ID', self.testO365TenantID), + ('84', 'Organization Name', self.testOrganizationIsO365Tenant), + ('85', 'MS Defender For Office365 Safe Links Version',self.testSafeLinksKeyVer), + # # These tests shall be the last ones. # @@ -1155,6 +1223,12 @@ class SMTPHeadersAnalysis: ('82', 'Header Containing Client IP', self.testAnyOtherIP), ) + ids = set() + + for test in (tests + testsDecodeAll + testsReturningArray): + assert test[0] not in ids, f"Test ID already taken: ({test[0]} - '{test[1]}')! IDs must be unique!" + ids.add(test[0]) + return (tests, testsDecodeAll, testsReturningArray) @staticmethod @@ -1372,10 +1446,10 @@ Results will be unsound. Make sure you have pasted your headers with correct spa idsOfDecodeAll = [int(x[0]) for x in testsDecodeAll] - for a in self.testsToRun: - if a in idsOfDecodeAll: - self.decode_all = True - break + #for a in self.testsToRun: + # if a in idsOfDecodeAll: + # self.decode_all = True + # break if self.decode_all: for testId, testName, testFunc in testsDecodeAll: @@ -1672,6 +1746,21 @@ Results will be unsound. Make sure you have pasted your headers with correct spa 'description' : '', } + def testSafeLinksKeyVer(self): + (num, header, value) = self.getHeader('X-MS-Exchange-Safelinks-Url-KeyVer') + if num == -1: return [] + + value = value.strip() + self.securityAppliances.add('MS Defender For Office365 - Safe Links') + result = f'- Microsoft Defender For Office365 (MDO) Safe Links was used in key version: {self.logger.colored(value, "green")}\n' + + + return { + 'header': header, + 'value' : value, + 'analysis' : result, + 'description' : '', + } def testSecurityAppliances(self): result = '' @@ -1679,6 +1768,11 @@ Results will be unsound. Make sure you have pasted your headers with correct spa self.logger.dbg('Spotted clues about security appliances:') + for (num, header, value) in self.headers: + for product, hdr in SMTPHeadersAnalysis.Security_Appliances_And_Their_Headers: + if re.search(re.escape(hdr), header, re.I): + self.securityAppliances.add(product) + for a in self.securityAppliances: parts = a.split(' ') skip = True @@ -1703,6 +1797,174 @@ Results will be unsound. Make sure you have pasted your headers with correct spa 'description' : '', } + def testO365TenantID(self): + (num, header, value) = self.getHeader('X-MS-Exchange-CrossTenant-Id') + if num == -1: return [] + + value = SMTPHeadersAnalysis.flattenLine(value).strip().replace(' ', '') + result = f'- Office365 Tenant ID: {self.logger.colored(value, "green")}\n' + + try: + r = requests.get(f'https://login.microsoftonline.com/{value}/.well-known/openid-configuration') + out = r.json() + + # + # Sample response for "microsoft.com": + # https://login.microsoftonline.com/microsoft.com/.well-known/openid-configuration + # + # RESPONSE: + # { + # "token_endpoint": "https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/oauth2/token", + # "token_endpoint_auth_methods_supported": [ + # "client_secret_post", + # "private_key_jwt", + # "client_secret_basic" + # ], + # "jwks_uri": "https://login.microsoftonline.com/common/discovery/keys", + # "response_modes_supported": [ + # "query", + # "fragment", + # "form_post" + # ], + # "subject_types_supported": [ + # "pairwise" + # ], + # "id_token_signing_alg_values_supported": [ + # "RS256" + # ], + # "response_types_supported": [ + # "code", + # "id_token", + # "code id_token", + # "token id_token", + # "token" + # ], + # "scopes_supported": [ + # "openid" + # ], + # "issuer": "https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/", + # "microsoft_multi_refresh_token": true, + # "authorization_endpoint": "https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/oauth2/authorize", + # "device_authorization_endpoint": "https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/oauth2/devicecode", + # "http_logout_supported": true, + # "frontchannel_logout_supported": true, + # "end_session_endpoint": "https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/oauth2/logout", + # "claims_supported": [ + # "sub", + # "iss", + # "cloud_instance_name", + # "cloud_instance_host_name", + # "cloud_graph_host_name", + # "msgraph_host", + # "aud", + # "exp", + # "iat", + # "auth_time", + # "acr", + # "amr", + # "nonce", + # "email", + # "given_name", + # "family_name", + # "nickname" + # ], + # "check_session_iframe": "https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/oauth2/checksession", + # "userinfo_endpoint": "https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/openid/userinfo", + # "kerberos_endpoint": "https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/kerberos", + # "tenant_region_scope": "WW", + # "cloud_instance_name": "microsoftonline.com", + # "cloud_graph_host_name": "graph.windows.net", + # "msgraph_host": "graph.microsoft.com", + # "rbac_url": "https://pas.windows.net" + # } + + if 'error' in out.keys() and out['error'] != '': + m = out['error'] + result += '\t- Office365 Tenant ' + self.logger.colored(f'does not exist: {m}\n', 'red') + else: + result += '\t- Office365 Tenant ' + self.logger.colored(f'exists.\n', 'green') + + tmp = '' + + num0 = 0 + for (num1, header1, value1) in self.headers: + value1 = SMTPHeadersAnalysis.flattenLine(value1).strip() + if value.lower() in value1.lower() and header1.lower() != header.lower(): + num0 += 1 + pos = value1.lower().find(value.lower()) + val = value1 + if pos != -1: + val = value1[:pos] + self.logger.colored(value1[pos:pos+len(value)], 'yellow') + value1[pos+len(value):] + + tmp += f'\t- ({num0:02}) Header: {header1}\n' + tmp += f'\t Value: {val}\n\n' + + if len(tmp) > 0: + result += '\n - Tenant ID found in following headers:\n' + result += '\n' + tmp + '\n' + + except: + self.logger.err(f'Could not fetch Office365 tenant OpenID configuration.') + result += self.logger.colored('\t- Error: Could not fetch information about Office365 Tenant.\n', 'red') + + return { + 'header': header, + 'value' : value, + 'analysis' : result, + 'description' : '', + } + + + def testOrganizationIsO365Tenant(self): + (num, header, value) = self.getHeader('X-OriginatorOrg') + if num == -1: return [] + + value = SMTPHeadersAnalysis.flattenLine(value).strip() + + result = f'- Organization name disclosed: {self.logger.colored(value, "green")}\n' + + try: + r = requests.get(f'https://login.microsoftonline.com/{value}/.well-known/openid-configuration') + out = r.json() + + if 'error' in out.keys() and out['error'] != '': + m = out['error'] + return [] + + result += '\n - Organization disclosed in "X-OriginatorOrg" is a valid Office 365 Tenant:\n' + tid = out['token_endpoint'].replace('https://login.microsoftonline.com/', '') + tid = tid.replace('/oauth2/token', '') + + result += '\t- Office365 Tenant ID: ' + self.logger.colored(tid, 'green') + '\n' + tmp = '' + + num0 = 0 + for (num1, header1, value1) in self.headers: + value1 = SMTPHeadersAnalysis.flattenLine(value1).strip() + if value.lower() in value1.lower() and header1.lower() != header.lower(): + num0 += 1 + pos = value1.lower().find(value.lower()) + val = value1 + if pos != -1: + val = value1[:pos] + self.logger.colored(value1[pos:pos+len(value)], 'yellow') + value1[pos+len(value):] + + tmp += f'\t- ({num0:02}) Header: {header1}\n' + tmp += f'\t Value: {val}\n\n' + + if len(tmp) > 0: + result += '\n - Organization name was also found in following headers:\n' + result += '\n' + tmp + '\n' + + except: + self.logger.err(f'Could not fetch Office365 tenant OpenID configuration.') + + return { + 'header': header, + 'value' : value, + 'analysis' : result, + 'description' : '', + } + def testXProofpointSpamDetails(self): (num, header, value) = self.getHeader('X-Proofpoint-Spam-Details') if num == -1: return [] @@ -3314,6 +3576,7 @@ Src: https://www.cisco.com/c/en/us/td/docs/security/esa/esa11-1/user_guide/b_ESA for prop in props: if prop in SMTPHeadersAnalysis.ATP_Message_Properties.keys(): result += f'\t- ' + self.logger.colored(SMTPHeadersAnalysis.ATP_Message_Properties[prop], 'magenta') + '\n' + self.securityAppliances.add('MS Defender For Office365 - ' + SMTPHeadersAnalysis.ATP_Message_Properties[prop]) return { 'header' : header, @@ -3721,12 +3984,12 @@ Src: https://www.cisco.com/c/en/us/td/docs/security/esa/esa11-1/user_guide/b_ESA result = '' - obj1 = self._parseBulk(num, header, value) - result += obj1['analysis'] - obj2 = self._parseAntiSpamReport(num, header, value) result += obj2['analysis'] + obj1 = self._parseBulk(num, header, value) + result += '\n' + obj1['analysis'] + self.securityAppliances.add('MS ForeFront Anti-Spam') obj['analysis'] = result @@ -4113,14 +4376,17 @@ Src: https://www.cisco.com/c/en/us/td/docs/security/esa/esa11-1/user_guide/b_ESA if num == -1: return [] parsed = {} - self.securityAppliances.add('MS ForeFront Anti-Spam') - result = '- This header denotes what to do with received message, where to put it.\n\n' + self.securityAppliances.add('MS ForeFront Anti-Spam') + result = '- This header denotes where to move received message. Informs about applied Mail Rules, target directory in user\'s Inbox.\n\n' for entry in value.split(';'): if len(entry.strip()) == 0: continue k, v = entry.split(':') if k not in parsed.keys(): - parsed[k] = v + parsed[k.lower()] = v + + if 'ucf' in parsed.keys() and 'dest' in parsed.keys() and parsed['ucf'] == '1' and parsed['dest'] == 'J': + result += self.logger.colored(f'- WARNING: User created a custom mail rule that moved this message to JUNK folder!\n', "red") for k, v in parsed.items(): elem = None @@ -4155,9 +4421,9 @@ Src: https://www.cisco.com/c/en/us/td/docs/security/esa/esa11-1/user_guide/b_ESA result += tmp + '\n' for k in ['SFS', 'RULEID', 'ENG']: - if k in parsed.keys(): + if k.lower() in parsed.keys(): res = '' - rules = [x.replace('(', '') for x in parsed[k].split(')')] + rules = [x.replace('(', '') for x in parsed[k.lower()].split(')')] if len(rules) == 1 and len(rules[0].strip()) == 0: rules = [] @@ -4590,6 +4856,7 @@ def printOutput(out): value = v['value'] analysis = analysis.strip() + if analysis.startswith('\n'): analysis[1:] value = str(textwrap.fill( v['value'], @@ -4628,7 +4895,8 @@ def printOutput(out): {value} {logger.colored("ANALYSIS", "yellow")}: - {analysis} + +{analysis} ''' else: output += f''' @@ -4636,7 +4904,8 @@ def printOutput(out): ({num}) Test: {logger.colored(k, "cyan")} {logger.colored("ANALYSIS", "yellow")}: - {analysis} + +{analysis} ''' if options['format'] == 'html':