Added YAML configuration and test code

config.yml

@@ -0,0 +1,35 @@
+## Logging configuration
+logging:
+  file_only: false
+  out_path: 'logs'
+  level: 'DEBUG'
+  # Number of hours each log file should last before rolling over.
+  rollover_after_hours: 6
+  # Number of log files to keep before deleting old ones.
+  # 0 = no limit
+  max_log_files: 0
+
+## Service spoofing configuration
+services:
+  # Path to nmap-service-probes.
+  probe_file_location: 'nmap-service-probes'
+
+  ############################################################################################################
+  ## Re-enabling honeypot services is not recommended and will increase detection rate from shodan/nmap/etc ##
+  ############################################################################################################
+  # List of service types to avoid spoofing.
+  # Defaults to 'honeypot' to reduce detection rates
+  # Items can be regular expressions and are case insensitive.
+  disabled_service_types: ['honeypot']
+
+  # List of product names to avoid spoofing.
+  # Defaults to patterns matching honeypots to reduce detection rates.
+  # Items can be regular expressions and are case insensitive.
+  disabled_product_names: ['.*honeypot.*', '.*honeyd.*', 'Dumbster fake smtpd', '.*nepenthes.*']
+
+## Networking-related items
+networking:
+  # Should be a port that does not have a service being spoofed on it, and is not being used currently.
+  real_port: 11337
+  max_ports_per_service: 10
+  max_replies: 10

liquid_honey.py

@@ -2,12 +2,11 @@
 import os
 import shutil
 import sys
-
-import requests
-from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
+from logging.handlers import TimedRotatingFileHandler
 
 import click
 
+from src.lh.config import LHConfig
 from src.lh.parse_nmap_probes import ProbeFileParser
 from src.lh.port_selector import PortSelector
 from src.lh.server.probe_server import ProbeServer
@@ -19,10 +18,10 @@
 @click.option('--listen-port', type=int, default=11337, help='Set a port to forward traffic to. '
                                                              'This should be a service you aren\'t spoofing. '
                                                              '(Default 1137)')
-@click.option('--create-rules', default=False, is_flag=True, help='Attempt to create iptables rules (requires root!)')
+@click.option('--create-rules', default=False, required=False, is_flag=True, help='Attempt to create iptables rules (requires root!)')
 def main(stdout, log_path, listen_port, create_rules):
     logging.basicConfig(format='[%(levelname)s] [%(asctime)s] %(message)s',
-                        filename='liquid-honey.log',
+                        filename=os.path.join(log_path, 'liquid-honey.log'),
                         level=logging.DEBUG)
     # Backups every 6 hrs, keeps up to 42 (7 days) worth of logs.
     rotator = TimedRotatingFileHandler(os.path.join(log_path, 'liquid-honey.log'),
@@ -36,11 +35,13 @@ def main(stdout, log_path, listen_port, create_rules):
 
     check_nmap_db()
 
-    configs = list(ProbeFileParser('nmap-service-probes').iter_parse())
-    configs = PortSelector(configs).config_iterator()
+    conf = LHConfig('config.yml')
+
+    probes = list(ProbeFileParser('nmap-service-probes').iter_parse())
+    probes = PortSelector(probes).config_iterator()
 
-    server = ProbeServer(listen_port, create_rules)
-    for port, config in configs:
+    server = ProbeServer(listen_port or conf.listen_port, conf.max_ports_per_service, conf.max_replies, create_rules)
+    for port, config in probes:
         server.add_from_config(port, config)
     server.run()
 

requirements.txt

@@ -1,2 +1,3 @@
 click
-exrex
+exrex
+pyyaml

src/lh/config.py

@@ -0,0 +1,55 @@
+import logging
+import re
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+import yaml
+
+
+class LHConfig(object):
+    PERMITTED_LOG_LEVELS = [
+        'CRITICAL', 'FATAL', 'ERROR', 'WARNING', 'WARN',
+        'INFO', 'DEBUG', 'NOTSET'
+    ]
+
+    def __init__(self, filename='config.yml'):
+        with open(filename) as f:
+            self.conf = yaml.load(f, loader=Loader)
+        self._parse_logging()
+
+    def _parse_logging(self):
+        # Logging
+        log_conf = self.conf.get('logging', {})
+        self.file_only = log_conf.get('file_only', False)
+        self.log_path = log_conf.get('out_path', 'logs')
+        self.log_level = log_conf.get('level', 'DEBUG')
+        self.log_rollover = log_conf.get('rollover_after_hours', 6)
+        self.max_log_files = log_conf.get('max_log_files', 42)
+
+        # Services
+        service_conf = self.conf.get('services', {})
+        self.service_probes_location = service_conf.get('probe_file_location', 'nmap-service-probes')
+
+        omit_services = service_conf.get("disabled_service_types", ["honeypot"])
+        self.omit_service_patterns = [re.compile(s, re.IGNORECASE) for s in omit_services]
+
+        omit_products = service_conf.get("disabled_product_names", [".*honeypot.*",
+                                                                    ".*honeyd.*",
+                                                                    "Dumbster fake smtpd",
+                                                                    ".*nepenthes.*"])
+        self.omit_product_patterns = [re.compile(p, re.IGNORECASE) for p in omit_products]
+
+        # Networking
+        networking_conf = self.conf.get('networking')
+        self.listen_port = networking_conf.get('real_port', 11337)
+        self.max_ports_per_service = networking_conf.get('max_ports_per_service', 10)
+        self.max_replies = networking_conf.get('max_replies', 10)
+
+    def get_log_level(self):
+        if not self.log_level.upper() in self.PERMITTED_LOG_LEVELS:
+            print("Unable to parse log level. Level '{}' not recognized."
+                  "Defaulting to DEBUG. Valid options: {}".format(self.log_level.upper(),
+                                                                  self.PERMITTED_LOG_LEVELS))
+            return logging.DEBUG
+        return getattr(logging, self.log_level.upper())

src/lh/parse_nmap_probes.py

@@ -76,6 +76,7 @@ def iter_parse(self):
                 if isinstance(parsed_directive, Match):
                     self._complete_match(parsed_directive)
                 self.cur_probe.add_directive(parsed_directive)
+        yield self.cur_probe
 
     def start_probe(self):
         if self.cur_probe:

src/lh/server/probe_server.py

@@ -1,19 +1,15 @@
 import sys
 import struct
 import subprocess
-from shutil import which
-
-from ssl import SSLContext
-if sys.version_info >= (3, 6):
-    from ssl import PROTOCOL_TLS
-else:
-    from ssl import PROTOCOL_TLSv1_2 as PROTOCOL_TLS
-
 import logging
 import select
 import socket
 import threading
 import traceback
+from shutil import which
+
+from ssl import SSLContext
+
 from abc import ABC
 from random import SystemRandom
 
@@ -22,18 +18,28 @@
 from src.lh.server.exception import SocketException
 from src.lh.service_directives import SoftMatch
 
+if sys.version_info >= (3, 6):
+    from ssl import PROTOCOL_TLS
+else:
+    from ssl import PROTOCOL_TLSv1_2 as PROTOCOL_TLS
+
 CLAIMED_PORTS = []
 
 
-class ProbeServer(ABC):
-    MAX_PORTS_PER_SERVER = 200
-    BUFFER_SIZE = 256
-    MAX_REPLIES = 10
+class ProbeServer(object):
     SO_ORIGINAL_DST = 80
+
+    max_ports_per_service = 200
+    BUFFER_SIZE = 256
+    max_replies = 10
     socket_threads = []
     ssl_context = None
 
-    def __init__(self, listen_port, create_rules):
+    def __init__(self, listen_port, max_ports_per_service, max_replies, create_rules):
+        self.max_replies = max_replies
+        self.max_ports_per_service = max_ports_per_service
+        self.listen_port = listen_port
+
         self.sockets = []
         self.ports = []
         self.fingerprint_to_probes = {}
@@ -42,7 +48,6 @@ def __init__(self, listen_port, create_rules):
         self.ssl = None
         self.rand = SystemRandom()
         self.create_rules = create_rules
-        self.listen_port = listen_port
         self.add_server(listen_port, False, False, '127.0.0.1')
 
     def _add_iptables_rule(self, is_udp, from_port, to_port):
@@ -103,7 +108,6 @@ def add_server(self, port, udp, ssl, hostname):
         self.socket_threads.append(server)
 
     def run(self):
-        # TODO: Multithread this
         while True:
             readable_streams, _, _ = select.select(self.socket_threads, [], [])
             server = readable_streams[0]
@@ -133,28 +137,28 @@ def handle_client(self, client, address):
                 port, srv_ip = struct.unpack("!2xH4s8x", dst)
                 logging.info("[%s:%s] -> S(%d): %s %s", address[0], address[1], port, str(data),
                              '(SSL)' if self.ssl else '')
-                
+
                 if port not in client_reply_map:
                     client_reply_map[port] = 0
-                elif client_reply_map[port] >= self.MAX_REPLIES:
-                    log.info('Client exceeded chatter for port {}. Killing connection...'.format(port))
-                    break;
+                elif client_reply_map[port] >= self.max_replies:
+                    logging.info('Client exceeded chatter for port {}. Killing connection...'.format(port))
+                    break
 
                 client_reply_map[port] += 1
                 matches = self.port_options[port]
                 match = self.rand.choice(matches)
                 pattern = match.pattern
                 if isinstance(match, SoftMatch):
-                    response = exrex.getone(pattern, limit=10000)
+                    response = exrex.getone(pattern, limit=1000)
                 else:
-                    response = exrex.getone(pattern, limit=10000)
+                    response = exrex.getone(pattern, limit=1000)
 
                 response = response.encode('utf-8').decode()
                 if is_udp:
                     client.sendto(response.encode(), address)
                 else:
                     client.send(response.encode())
-            except (SocketException, ConnectionResetError, BrokenPipeError):
+            except (SocketException, ConnectionResetError, BrokenPipeError) as e:
                 client.close()
                 return False
             except Exception as e:

src/test/resources/config.yml

@@ -0,0 +1,35 @@
+## Logging configuration
+logging:
+  file_only: false
+  out_path: 'logs'
+  level: 'DEBUG'
+  # Number of hours each log file should last before rolling over.
+  rollover_after_hours: 6
+  # Number of log files to keep before deleting old ones.
+  # 0 = no limit
+  max_log_files: 0
+
+## Service spoofing configuration
+services:
+  # Path to nmap-service-probes.
+  probe_file_location: 'nmap-service-probes'
+
+  ############################################################################################################
+  ## Re-enabling honeypot services is not recommended and will increase detection rate from shodan/nmap/etc ##
+  ############################################################################################################
+  # List of service types to avoid spoofing.
+  # Defaults to 'honeypot' to reduce detection rates
+  # Items can be regular expressions and are case insensitive.
+  disabled_service_types: ['honeypot']
+
+  # List of product names to avoid spoofing.
+  # Defaults to patterns matching honeypots to reduce detection rates.
+  # Items can be regular expressions and are case insensitive.
+  disabled_product_names: ['.*honeypot.*', '.*honeyd.*', 'Dumbster fake smtpd', '.*nepenthes.*']
+
+## Networking-related items
+networking:
+  # Should be a port that does not have a service being spoofed on it, and is not being used currently.
+  real_port: 11337
+  max_ports_per_service: 10
+  max_replies: 10

src/test/resources/honeypot-strings.txt

@@ -0,0 +1,14 @@
+Network Flight Recorder BackOfficer Friendly http honeypot
+Dionaea honeypothttpd
+Nepenthes honeypot netbios-ssn httpd
+Honeyd
+Kojoney SSH honeypot
+SSHTroll ssh honeypot
+Network Flight Recorder BackOfficer Friendly honeypot
+Network Flight Recorder BackOfficer Friendly telnet honeypot
+honeyd cmdexe.pl
+Nepenthes honeypot netbios-ssn
+Dionaea honeypot smbd
+Dionaea Honeypot httpd
+Dionaea Honeypot sipd
+Dionaea honeypot MS-SQL server

src/test/resources/test_directives.txt

@@ -0,0 +1,3 @@
+Probe TCP GetRequest q|GET / HTTP/1.0\r\n\r\n|
+ports 22
+match test-service m|^some-data$| p/Test Service/

src/test/test_honeypot_patterns.py

@@ -0,0 +1,29 @@
+from src.lh.config import LHConfig
+
+
+class TestConfigValues(object):
+    @classmethod
+    def setup_class(cls):
+        cls.conf = LHConfig('config.yml')
+
+    def test_config_categories(self):
+        assert self.conf.file_only == False
+        assert self.conf.service_probes_location == 'nmap-service-probes'
+        assert self.conf.service_probes_location == 'nmap-service-probes'
+        assert self.conf.listen_port == 11337
+
+    def test_log_lvl(self):
+        # logging.DEBUG == 10
+        assert self.conf.get_log_level() == 10
+
+    def test_honeypot_patterns(self):
+        with open('honeypot-strings.txt') as f:
+            honeypot_services = f.readlines()
+
+        for service in honeypot_services:
+            one_matches = False
+            for pattern in self.conf.omit_product_patterns:
+                if pattern.match(service):
+                    one_matches = True
+                    break
+            assert one_matches