diff --git a/README.md b/README.md index 84d0e7d..9b696e3 100644 --- a/README.md +++ b/README.md @@ -22,4 +22,9 @@ Pastehunter supports several output modules: - Dump to CSV file. - Send to syslog. + ## Supported Sandboxes + Pastehunter supports several sandboxes that decoded data can be sent to: + - Cuckoo + - Viper + For examples of data discovered using pastehunter check out my posts https://techanarchy.net/blog/hunting-pastebin-with-pastehunter and https://techanarchy.net/blog/pastehunter-the-results diff --git a/docs/postprocess.rst b/docs/postprocess.rst index eaac1d5..15512d9 100644 --- a/docs/postprocess.rst +++ b/docs/postprocess.rst @@ -29,20 +29,7 @@ when the full paste is a base64 blob, i.e. it will not extract base64 code that - **rule_list**: List of rules that will trigger the postprocess module. - -Cuckoo -^^^^^^ -If the samples match a binary file format you can optionaly send the file for analysis by a Cuckoo Sandbox. - -- **api_host**: IP or hostname for a Cuckoo API endpoint. -- **api_port**: Port number for a Cuckoo API endpoint. - -Viper -^^^^^ -If the samples match a binary file format you can optionaly send the file to a Viper instance for further analysis. - -- **api_host**: IP or hostname for a Cuckoo API endpoint. -- **api_port**: Port number for a Cuckoo API endpoint. +See the `Sandboxes documentation `_ for information on how to configure the sandboxes used for scanning decoded base64 data. Entropy diff --git a/docs/sandboxes.rst b/docs/sandboxes.rst new file mode 100644 index 0000000..07d099c --- /dev/null +++ b/docs/sandboxes.rst @@ -0,0 +1,25 @@ +Sandboxes +========= + +There are a few sandboxes that can be configured and used in various post process steps. + +There are a few generic options for each input. + +- **enabled**: This turns the sandbox on and off. +- **module**: This is used internally by pastehunter. + +Cuckoo +------ + +If the samples match a binary file format you can optionaly send the file for analysis by a Cuckoo Sandbox. + +- **api_host**: IP or hostname for a Cuckoo API endpoint. +- **api_port**: Port number for a Cuckoo API endpoint. + +Viper +----- + +If the samples match a binary file format you can optionaly send the file to a Viper instance for further analysis. + +- **api_host**: IP or hostname for a Viper API endpoint. +- **api_port**: Port number for a Viper API endpoint. diff --git a/pastehunter.py b/pastehunter.py index 25877d2..8a1f796 100644 --- a/pastehunter.py +++ b/pastehunter.py @@ -157,7 +157,7 @@ def paste_scanner(): sleep(0.5) else: paste_data = q.get() - with timeout(seconds=10): + with timeout(seconds=conf['general']['process_timeout']): # Start a timer start_time = time.time() logger.debug("Found New {0} paste {1}".format(paste_data['pastesite'], paste_data['pasteid'])) @@ -237,6 +237,17 @@ def paste_scanner(): # Else use the rule name else: results.append(match.rule) + + # Store additional fields for passing on to post processing + encoded_paste_data = raw_paste_data.encode('utf-8') + md5 = hashlib.md5(encoded_paste_data).hexdigest() + sha256 = hashlib.sha256(encoded_paste_data).hexdigest() + paste_data['MD5'] = md5 + paste_data['SHA256'] = sha256 + paste_data['raw_paste'] = raw_paste_data + paste_data['YaraRule'] = results + # Set the size for all pastes - This will override any size set by the source + paste_data['size'] = len(raw_paste_data) # Store all OverRides other options. paste_site = paste_data['confname'] @@ -282,21 +293,6 @@ def paste_scanner(): results.append('no_match') if len(results) > 0: - - encoded_paste_data = raw_paste_data.encode('utf-8') - md5 = hashlib.md5(encoded_paste_data).hexdigest() - sha256 = hashlib.sha256(encoded_paste_data).hexdigest() - paste_data['MD5'] = md5 - paste_data['SHA256'] = sha256 - # It is possible a post module modified or set this field. - if not paste_data.get('raw_paste'): - paste_data['raw_paste'] = raw_paste_data - paste_data['size'] = len(raw_paste_data) - else: - # Set size based on modified value - paste_data['size'] = len(paste_data['raw_paste']) - - paste_data['YaraRule'] = results for output in outputs: try: output.store_paste(paste_data) diff --git a/postprocess/post_b64.py b/postprocess/post_b64.py index 150d671..260e16e 100644 --- a/postprocess/post_b64.py +++ b/postprocess/post_b64.py @@ -1,9 +1,8 @@ -import io import re import hashlib +import importlib import gzip import logging -import requests from base64 import b64decode # This gets the raw paste and the paste_data json object from common import parse_config @@ -45,6 +44,7 @@ def run(results, raw_paste_data, paste_object): paste_object["decompressed_stream"] = encoded except Exception as e: logger.error("Unable to decompress gzip stream") + if rule == 'b64_exe': try: raw_exe = b64decode(raw_paste_data) @@ -55,47 +55,18 @@ def run(results, raw_paste_data, paste_object): # We are guessing that the sample has been submitted, and crafting a URL paste_object["VT"] = 'https://www.virustotal.com/#/file/{0}'.format(paste_object["exe_md5"]) - # Cuckoo - if conf["post_process"]["post_b64"]["cuckoo"]["enabled"]: - logger.info("Submitting to Cuckoo") - try: - task_id = send_to_cuckoo(raw_exe, paste_object["pasteid"]) - paste_object["Cuckoo Task ID"] = task_id - logger.info("exe submitted to Cuckoo with task id {0}".format(task_id)) - except Exception as e: - logger.error("Unabled to submit sample to cuckoo") - - # Viper - if conf["post_process"]["post_b64"]["viper"]["enabled"]: - send_to_cuckoo(raw_exe, paste_object["pasteid"]) - - # VirusTotal + # If sandbox modules are enabled then submit the file + for sandbox, sandbox_values in conf["sandboxes"].items(): + if sandbox_values["enabled"]: + logger.info("Uploading file {0} using {1}".format(paste_object["pasteid"], sandbox_values["module"])) + sandbox_module = importlib.import_module(sandbox_values["module"]) + paste_object = sandbox_module.upload_file(raw_exe, paste_object) except Exception as e: logger.error("Unable to decode exe file") - # Get unique domain count # Update the json # Send the updated json back return paste_object - - -def send_to_cuckoo(raw_exe, pasteid): - cuckoo_ip = conf["post_process"]["post_b64"]["cuckoo"]["api_host"] - cuckoo_port = conf["post_process"]["post_b64"]["cuckoo"]["api_port"] - cuckoo_host = 'http://{0}:{1}'.format(cuckoo_ip, cuckoo_port) - submit_file_url = '{0}/tasks/create/file'.format(cuckoo_host) - files = {'file': ('{0}.exe'.format(pasteid), io.BytesIO(raw_exe))} - submit_file = requests.post(submit_file_url, files=files).json() - task_id = None - try: - task_id = submit_file['task_id'] - except KeyError: - try: - task_id = submit_file['task_ids'][0] - except KeyError: - logger.error(submit_file) - - return task_id diff --git a/sandboxes/__init__.py b/sandboxes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sandboxes/cuckoo.py b/sandboxes/cuckoo.py new file mode 100644 index 0000000..7685646 --- /dev/null +++ b/sandboxes/cuckoo.py @@ -0,0 +1,36 @@ +import io +import logging +import requests +from common import parse_config +conf = parse_config() + +logger = logging.getLogger('pastehunter') + +def upload_file(raw_file, paste_object): + try: + task_id = send_to_cuckoo(raw_file, paste_object["pasteid"]) + paste_object["Cuckoo Task ID"] = task_id + logger.info("exe submitted to Cuckoo with task id {0}".format(task_id)) + except Exception as e: + logger.error("Unabled to submit sample to cuckoo") + + # Send any updated json back + return paste_object + +def send_to_cuckoo(raw_exe, pasteid): + cuckoo_ip = conf["sandboxes"]["cuckoo"]["api_host"] + cuckoo_port = conf["sandboxes"]["cuckoo"]["api_port"] + cuckoo_host = 'http://{0}:{1}'.format(cuckoo_ip, cuckoo_port) + submit_file_url = '{0}/tasks/create/file'.format(cuckoo_host) + files = {'file': ('{0}.exe'.format(pasteid), io.BytesIO(raw_exe))} + submit_file = requests.post(submit_file_url, files=files).json() + task_id = None + try: + task_id = submit_file['task_id'] + except KeyError: + try: + task_id = submit_file['task_ids'][0] + except KeyError: + logger.error(submit_file) + + return task_id diff --git a/sandboxes/viper.py b/sandboxes/viper.py new file mode 100644 index 0000000..b7f085d --- /dev/null +++ b/sandboxes/viper.py @@ -0,0 +1,19 @@ +import io +import logging +import requests +from common import parse_config +conf = parse_config() + +logger = logging.getLogger('pastehunter') + +def upload_file(raw_file, paste_object): + viper_ip = conf["sandboxes"]["viper"]["api_host"] + viper_port = conf["sandboxes"]["viper"]["api_port"] + viper_host = 'http://{0}:{1}'.format(viper_ip, viper_port) + + submit_file_url = '{0}/tasks/create/file'.format(viper_host) + files = {'file': ('{0}.exe'.format(paste_object["pasteid"]), io.BytesIO(raw_file))} + submit_file = requests.post(submit_file_url, files=files).json() + + # Send any updated json back + return paste_object diff --git a/settings.json.sample b/settings.json.sample index f006a2e..f2cc1b0 100644 --- a/settings.json.sample +++ b/settings.json.sample @@ -153,7 +153,22 @@ "format": "%(asctime)s [%(threadName)-12.12s] %(levelname)s:%(message)s" }, "general": { - "run_frequency": 300 + "run_frequency": 300, + "process_timeout": 5 + }, + "sandboxes": { + "cuckoo": { + "enabled": false, + "module": "sandboxes.cuckoo", + "api_host": "127.0.0.1", + "api_port": 8080 + }, + "viper": { + "enabled": false, + "module": "sandboxes.viper", + "api_host": "127.0.0.1", + "api_port": 8080 + } }, "post_process": { "post_email": { @@ -164,17 +179,7 @@ "post_b64": { "enabled": true, "module": "postprocess.post_b64", - "rule_list": ["b64_exe", "b64_rar", "b64_zip", "b64_gzip"], - "cuckoo": { - "enabled": false, - "api_host": "127.0.0.1", - "api_port": 8080 - }, - "viper": { - "enabled": false, - "api_host": "127.0.0.1", - "api_port": 8080 - } + "rule_list": ["b64_exe", "b64_rar", "b64_zip", "b64_gzip"] }, "post_entropy": { "enabled": false,