Updates

correlate-rules.py

@@ -11,10 +11,14 @@ import glob
 import base64
 
 rules = {}
+files_and_their_rules = {}
 scanned = set()
 
+FILES_PREFIX='analysis-'
+
 def walk(path):
     global rules
+    global files_and_their_rules
     global scanned
 
     print(f'Walking {path}...')
@@ -23,7 +27,14 @@ def walk(path):
         if not file.lower().endswith('.txt'):
             continue
 
-        if file in scanned: continue
+        if file in scanned: 
+            continue
+
+        base = os.path.basename(file)
+        if len(FILES_PREFIX) > 0:
+            if not base.lower().startswith(FILES_PREFIX.lower()):
+                continue
+
         scanned.add(file)
 
         data = ''
@@ -33,6 +44,11 @@ def walk(path):
         for m in re.finditer(r'(\(\d{4,}\))', data, re.I):
             rule = m.group(1)
 
+            if file not in files_and_their_rules.keys():
+                files_and_their_rules[file] = set()
+
+            files_and_their_rules[file].add(rule)
+
             if rule in rules.keys():
                 if file not in rules[rule]['files']:
                     rules[rule]['count'] += 1
@@ -57,7 +73,6 @@ def main(argv):
 
     print(f'[.] Found {len(rules)} unique rules.:')
 
-    candidates = []
     for k, v in rules.items():
         if v['count'] > 1:
             print(f'\n\t- {k: <15}: occurences: {v["count"]} - files: {len(v["files"])}')
@@ -66,5 +81,38 @@ def main(argv):
                 for f in v['files']:
                     print('\t\t- ' + str(f))
 
+
+    output = ' #  | file1                                              | file2                                              |\n'
+    output+= '----+----------------------------------------------------+----------------------------------------------------+\n'
+
+    checked = set()
+    for k, v in files_and_their_rules.items():
+        for k1, v1 in files_and_their_rules.items():
+            if k == k1: 
+                continue
+
+            n = max(len(v.difference(v1)), len(v1.difference(v)))
+            if n <= 3 and n > 0:
+                if k not in checked and k1 not in checked:
+                    output += f' {n: <2} | {k[-50:]: <50} | {k1[-50:]: <50} |\n'
+                    checked.add(k)
+                    checked.add(k1)
+                    
+    output+= '----+----------------------------------------------------+----------------------------------------------------+\n'
+
+    print('\nCross-File rules differences:\n')
+    print(output)
+
+    print('\n\nFiles and rules matched:\n')
+
+    num = 0
+    s = {k: v for k, v in sorted(files_and_their_rules.items(), key=lambda item: len(item[1]))}
+
+    for k, v in s.items():
+        num += 1
+        print(f'{num: <3}. Rules: {len(v): <2}, File: {k}')
+
+    print()
+
 if __name__ == '__main__':
     main(sys.argv)