SpyGuard/analysis/classes/engine.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import json
import os
import re
import subprocess as sp
import sys
import time
from datetime import datetime
from ipaddress import IPv4Address, IPv6Address

import ssl
import socket
import OpenSSL
import requests

import pydig
import whois
from publicsuffix2 import get_sld
from netaddr import IPAddress, IPNetwork
from classes.jarm import get_jarm
from utils import get_config, get_iocs, get_whitelist


class Engine():

    def __init__(self, capture_directory):

        # Set some vars.
        self.analysis_start = datetime.now()
        self.connected = self.check_internet()
        self.working_dir = capture_directory
        self.assets_dir = f"{capture_directory}/assets/"
        self.rules_file = "/tmp/rules.rules"
        self.pcap_path = os.path.join(self.working_dir, "capture.pcap")
        self.records = []
        self.alerts = []
        self.dns = []
        self.files = []
        self.whitelist = []
        self.uncategorized = []
        self.analysed = []
        self.dns_failed = []
        self.dns_checked = []
        self.cert_checked = []
        self.errors = []
        self.analysis_end = None

        # Get configuration
        self.heuristics_analysis = get_config(("analysis", "heuristics"))
        self.iocs_analysis = get_config(("analysis", "iocs"))
        self.whitelist_analysis = get_config(("analysis", "whitelist"))
        self.active_analysis = get_config(("analysis", "active"))
        self.userlang = get_config(("frontend", "user_lang"))
        self.max_ports = get_config(("analysis", "max_ports"))
        self.http_default_ports = get_config(("analysis", "http_default_ports"))
        self.tls_default_ports = get_config(("analysis", "tls_default_ports"))
        self.free_issuers = get_config(("analysis", "free_issuers"))
        self.max_alerts = get_config(("analysis", "max_alerts"))
        self.indicators_types = get_config(("analysis", "indicators_types"))
        
        # Save detection methods used.
        self.detection_methods = { "iocs" : self.iocs_analysis, 
                                   "heuristics" : self.heuristics_analysis, 
                                   "active" : self.active_analysis }

        # Retreive IOCs.
        if self.iocs_analysis:
            self.bl_cidrs = [[IPNetwork(cidr[0]), cidr[1]] for cidr in get_iocs("cidr")]
            self.bl_hosts = get_iocs("ip4addr") + get_iocs("ip6addr")
            self.tor_nodes = self.get_tor_nodes()
            self.bl_domains = get_iocs("domain")
            self.bl_freedns = get_iocs("freedns")
            self.bl_certs = get_iocs("sha1cert")
            self.bl_jarms = get_iocs("jarm")
            self.bl_nameservers = get_iocs("ns")
            self.bl_tlds = get_iocs("tld")

        # Retreive whitelisted items.
        if self.whitelist_analysis:
            self.wl_cidrs = [IPNetwork(cidr) for cidr in get_whitelist("cidr")]
            self.wl_hosts = get_whitelist("ip4addr") + get_whitelist("ip6addr") + self.get_public_ip()
            self.wl_domains = get_whitelist("domain")

        # Load template language
        if not re.match("^[a-z]{2,3}$", self.userlang): self.userlang = "en"
       
        with open(os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "locales/{}.json".format(self.userlang))) as f:
            self.template = json.load(f)["alerts"]

    def check_internet(self) -> bool:
        """Check the internet link just with a small http request
        to an URL present in the configuration. If the link is down,
        retry 3 times.

        Returns:
            bool: True if everything works.
        """
        attempts = 3

        while True:
            try:
                url = get_config(("network", "internet_check"))
                requests.get(url, timeout=3)
                return True
            except:
                if attempts == 0:
                    return False
                else:
                    time.sleep(5)
                    attempts -= 1

    def get_public_ip(self) -> list:
        """Get the public IP address

        Returns:
            list: list containing the public IP address.
        """
        if self.connected:
            try:
                return [requests.get("https://api.ipify.org", timeout=3).text]
            except:
                return []
        else:
            return []

    def start_engine(self):
        """ This method starts suricata and then launch the 
            parsers to analyse the output logs. 
        """

        # Parse the eve.json file.
        self.parse_eve_file()

        # For each type of records, check it against heuristics.
        for record in self.records: 
            if self.whitelist_analysis: self.check_whitelist(record)
            self.check_domains(record)
            self.check_flow(record)
            self.check_tls(record)
            self.check_http(record)

        # Check for failed DNS answers (if spyguard not connected)
        for dnsname in list(set(self.dns_failed)):
            self.check_dnsname(dnsname)
        
    def parse_eve_file(self):
        """This method parses the eve.json file produced by suricata.
           For each record, it look at the record type and then append the self.record
           dictionnary which contains valuable data to look at suspicious stuff.
        """
        for record in open(f"{self.assets_dir}eve.json", "r").readlines():
            record = json.loads(record)
            try:
                if "flow" in record:
                    if "app_proto" not in record: record["app_proto"] = "failed"
                    proto = { "name" : record["app_proto"].upper() if record["app_proto"] != "failed" else record["proto"].upper(), "port" : record["dest_port"] if "dest_port" in record else -1 }
                   
                    if record["dest_ip"] not in [r["ip_dst"] for r in self.records]:
                        self.records.append({
                            "ip_dst" : record["dest_ip"],
                            "whitelisted" : False,
                            "suspicious" : False,
                            "protocols" : [proto],
                            "domains" : [],
                            "certificates" : []
                        })
                    else:
                        for rec in self.records:
                            if record["dest_ip"] == rec["ip_dst"]:
                                if proto not in rec["protocols"]:
                                    rec["protocols"].append(proto)
            except Exception as e:
                self.errors.append(f"Issue when processing the following eve record (flow): {json.dumps(record)}")

        for record in open(f"{self.assets_dir}eve.json", "r").readlines():
            record = json.loads(record)
            try:
                if "tls" in record:
                    for rec in self.records:
                        if record["dest_ip"] == rec["ip_dst"]:
                            if "version" in record["tls"]: 
                                if float(record["tls"]["version"].split(" ")[1]) < 1.3 and not "session_resumed" in record["tls"]:
                                    if record["tls"] not in rec["certificates"]:
                                        record["tls"]["port"] = record["dest_port"]
                                        rec["certificates"].append(record["tls"])
                                else:
                                    if "sni" in record["tls"] and record["tls"]["sni"] not in [c["sni"] for c in rec["certificates"]]:
                                        rec["certificates"].append({ "sni" : record["tls"]["sni"], "version" : record["tls"]["version"], "port" : record["dest_port"] })
                                    else:
                                        rec["certificates"].append({ "version" : record["tls"]["version"], "port" : record["dest_port"] })
            except Exception as e:
                self.errors.append(f"Issue when processing the following eve record (tls): {json.dumps(record)}")        

        for record in open(f"{self.assets_dir}eve.json", "r").readlines():
            record = json.loads(record)
            try:
                if "http" in record:
                    for rec in self.records:
                        if record["dest_ip"] == rec["ip_dst"]:
                            d = { "hostname" : record["http"]["hostname"] }
                            if "http_user_agent" in record["http"]:
                                d["user-agent"] = record["http"]["http_user_agent"]
                            if "http" in rec:
                                if not d in rec["http"]:
                                    rec["http"].append(d)
                            else:
                                rec["http"] = [d]
            except Exception as e:
                self.errors.append(f"Issue when processing the following eve record (http): {json.dumps(record)}")        

        for record in open(f"{self.assets_dir}eve.json", "r").readlines():
            record = json.loads(record)
            try:
                if "dns" in record:
                    if record["dns"]["type"] == "answer":
                        for rec in self.records:
                            if record["dns"]["rcode"] == "NOERROR":
                                if "grouped" in record["dns"]:
                                    if "A" in record["dns"]["grouped"] and rec["ip_dst"] in record["dns"]["grouped"]["A"]:
                                        if record["dns"]["rrname"] not in rec["domains"]:
                                            rec["domains"].append(record["dns"]["rrname"])
                                    elif "AAAA" in record["dns"]["grouped"] and rec["ip_dst"] in record["dns"]["grouped"]["AAAA"]:
                                        if record["dns"]["rrname"] not in rec["domains"]:
                                            rec["domains"].append(record["dns"]["rrname"])
                            elif record["dns"]["rcode"] == "SERVFAIL":
                                self.dns_failed.append(record["dns"]["rrname"])
            except Exception as e:
                self.errors.append(f"Issue when processing the following eve record (dns answer): {json.dumps(record)}")

        # This pass is if SpyGuard is not connected to Internet.
        # We still analyze the un answered DNS queries.
        for record in open(f"{self.assets_dir}eve.json", "r").readlines():
            record = json.loads(record)
            try:
                if "dns" in record:
                    if record["dns"]["type"] == "query":
                        if record["dns"]["rrname"] not in sum([r["domains"] for r in self.records], []):
                             self.records.append({
                                "ip_dst" : "--",
                                "whitelisted" : False,
                                "suspicious" : False,
                                "protocols" : [{"name" : "DNS", "port" : "53"}],
                                "domains" : [record["dns"]["rrname"]],
                                "certificates" : []
                            })
            except Exception as e:
                self.errors.append(f"Issue when processing the following eve record (dns query): {json.dumps(record)}")        

        for record in open(f"{self.assets_dir}eve.json", "r").readlines():
            record = json.loads(record)
            try:
                if "alert" in record and record["event_type"] == "alert":
                    for rec in self.records:
                        if record["dest_ip"] == rec["ip_dst"]:
                            rec["suspicious"] = True
                            self.alerts.append({"title": self.template["SNORT-01"]["title"].format(record["alert"]["signature"]),
                                                "description": self.template["SNORT-01"]["description"].format(rec["ip_dst"]),
                                                "host": rec["ip_dst"],
                                                "level": "High",
                                                "id": "SNORT-01"})

            except Exception as e:
                self.errors.append(f"Issue when processing the following eve record (dns answer): {json.dumps(record)}")


    def check_whitelist(self, record):
        """ This method is asked on each record. It:

            1. Check if the associated IP(v4/6) Address can be whitelisted 
            2. Check if one of the associated domain names can be whitelisted. 

            If its the case, the "whitelisted" key of the record is set to True. 
            Therefore, the record will be ignored for the rest of the analysis.  
        Args:
            record (dict): record to be processed. 
        """

        try: 
            assert IPv4Address(record["ip_dst"])

            if IPv4Address('224.0.0.0') <= IPv4Address(record["ip_dst"]) <= IPv4Address('239.255.255.255'):
                record["whitelisted"] = True
                return 

            for cidr in self.wl_cidrs:
                if IPAddress(record["ip_dst"]) in cidr:
                    record["whitelisted"] = True
                    return 

            for ip in self.wl_hosts:
                if record["ip_dst"] == ip:
                    record["whitelisted"] = True
                    return 
        except:
            pass
        
        try:
            assert IPv6Address(record["ip_dst"])

            if [record["ip_dst"].startswith(prefix) for prefix in ["fe80", "fc00", "ff02"]]: 
                record["whitelisted"] = True
                return

            for ip in self.wl_hosts:
                if record["ip_dst"] == ip:
                    record["whitelisted"] = True
                    return
        except:
            pass
        
        # We check if at least one of the associated 
        # domains is whitelisted
        for dom in self.wl_domains:
            for domain in record["domains"]:
                if domain.endswith(dom):
                    record["whitelisted"] = True
                    return
        
    def check_domains(self, record):
        """Check the domains associated to each record.
           First this method checks if the record is whitelisted. If not:
              1. Leverage a low alert if the record don't have any associated DNSName
              2. Check each domain associated to the record by calling check_dnsname.
        Args:
            record (dict): record to be processed.
        """
        if record["whitelisted"]: return 

        if self.heuristics_analysis:
            # Otherwise, we alert the user that an IP haven't been resolved by 
            # a DNS answer during the session...
            if record["domains"] == []:
                record["suspicious"] = True
                self.alerts.append({"title": self.template["PROTO-05"]["title"].format(record["ip_dst"]),
                                    "description": self.template["PROTO-05"]["description"].format(record["ip_dst"]),
                                    "host": record["ip_dst"],
                                    "level": "Low",
                                    "id": "PROTO-05"})

        # Check each associated domain.
        for domain in record["domains"]:
            if self.check_dnsname(domain): 
                record["suspicious"] = True

    def check_dnsname(self, dnsname):
        """Check a domain name against a set of IOCs / heuristics.
              1. Check if the parent domain is blacklisted. 
              2. Check if the parent domain is a Free DNS.
              3. Check if the domain extension is a suspicious TLD.
              4. Check if the name servers associated to the domain are suspicious.
              5. Check if the domain have been registered recently - less than one year.
        Args:
            record (dict): record to be processed.
        Returns:
            supicious (bool) : if an alert has been leveraged. 
        """
        suspicious = False
        
        if self.iocs_analysis:
            for domain in self.bl_domains:
                if dnsname.endswith(domain[0]) and any(t in self.indicators_types for t in [domain[1], "all"]):
                    if domain[1] == "dual": 
                        suspicious = True
                        self.alerts.append({"title": self.template["IOC-12"]["title"],
                                            "description": self.template["IOC-12"]["description"].format(domain[0]),
                                            "host": domain[0],
                                            "level": "Low",
                                            "id": "IOC-12"})
                    elif domain[1] == "tracker": 
                        suspicious = True
                        self.alerts.append({"title": self.template["IOC-04"]["title"].format(domain[0], "tracker"),
                                            "description": self.template["IOC-04"]["description"].format(domain[0], "tracker"),
                                            "host": domain[0],
                                            "level": "Low",
                                            "id": "IOC-04"})
                    elif domain[1] == "doh": 
                        suspicious = True
                        self.alerts.append({"title": self.template["IOC-13"]["title"].format(f"{dnsname}"),
                                                "description": self.template["IOC-13"]["description"].format(f"{dnsname}"),
                                                "host": dnsname,
                                                "level": "Low",
                                                "id": "IOC-13"})
                    else:
                        suspicious = True
                        self.alerts.append({"title": self.template["IOC-03"]["title"].format(dnsname, domain[1].upper()),
                                            "description": self.template["IOC-03"]["description"].format(dnsname),
                                            "host": dnsname,
                                            "level": "High",
                                            "id": "IOC-03"})
            for domain in self.bl_freedns:
                if dnsname.endswith(domain[0]) and any(t in self.indicators_types for t in [domain[1], "all"]):
                    suspicious = True
                    self.alerts.append({"title": self.template["IOC-05"]["title"].format(dnsname),
                                        "description": self.template["IOC-05"]["description"].format(dnsname),
                                        "host": dnsname,
                                        "level": "Moderate",
                                        "id": "IOC-05"})
                    
        if self.heuristics_analysis:        
            for domain in self.bl_tlds:
                if dnsname.endswith(domain[0]) and any(t in self.indicators_types for t in [domain[1], "all"]):
                    suspicious = True
                    self.alerts.append({"title": self.template["IOC-06"]["title"].format(dnsname),
                                        "description": self.template["IOC-06"]["description"].format(dnsname, domain[0]),
                                        "host": dnsname,
                                        "level": "Low",
                                        "id": "IOC-06"})
                    
        if self.active_analysis and self.connected:
            domain = get_sld(dnsname)
            if domain not in self.dns_checked:
                self.dns_checked.append(domain)
                try:
                    name_servers = pydig.query(domain, "NS")
                    if len(name_servers):
                        for ns in self.bl_nameservers:
                            if name_servers[0].endswith(".{}.".format(ns[0])) and any(t in self.indicators_types for t in [ns[1], "all"]):
                                suspicious = True
                                self.alerts.append({"title": self.template["ACT-01"]["title"].format(dnsname, name_servers[0]),
                                                    "description": self.template["ACT-01"]["description"].format(dnsname),
                                                    "host": dnsname,
                                                    "level": "Moderate",
                                                    "id": "ACT-01"})
                except Exception as e:
                    self.errors.append(f"Issue when doing a dig NS query to {domain}, are you connected? Error: {str(e)}")        

                try:
                    whois_record = whois.whois(domain)
                    creation_date = whois_record.creation_date if type(whois_record.creation_date) is not list else whois_record.creation_date[0]
                    creation_days = abs((datetime.now() - creation_date).days)
                    if creation_days < 365:
                        suspicious = True
                        self.alerts.append({"title": self.template["ACT-02"]["title"].format(dnsname, creation_days),
                                            "description": self.template["ACT-02"]["description"].format(dnsname),
                                            "host": dnsname,
                                            "level": "Moderate",
                                            "id": "ACT-02"})
                except Exception as e:
                    self.errors.append(f"Issue when doing a WHOIS query to {domain}, are you connected? Error: {str(e)}")
        
        return suspicious
        

    def check_flow(self, record):
        """Check a network flow against a set of IOCs / heuristics.
              1. Check if the IP Address is blacklisted 
              2. Check if the IP Address is inside a blacklisted CIDR
              3. Check if the UDP or ICMP protocol is going outside of the local network. 
              4. Check if the HTTP protocol is not using default HTTP ports.
              5. Check if the network flow is using a port > 1024.
        Args:
            record (dict): record to be processed.
        Returns:
            supicious (bool) : if an alert has been leveraged. 
        """
        if record["whitelisted"]: return 

        resolved_host = record["domains"][0] if len(record["domains"]) else record["ip_dst"]

        if self.iocs_analysis:
            for host in self.bl_hosts:
                if record["ip_dst"] == host[0] and any(t in self.indicators_types for t in [host[1], "all"]):
                    if host[1] == "dual": 
                        record["suspicious"] = True
                        self.alerts.append({"title": self.template["IOC-12"]["title"],
                                            "description": self.template["IOC-12"]["description"].format(resolved_host),
                                            "host": resolved_host,
                                            "level": "Low",
                                            "id": "IOC-12"})
                    if host[1] == "tracker": 
                        record["suspicious"] = True
                        self.alerts.append({"title": self.template["IOC-04"]["title"].format(resolved_host, "tracker"),
                                            "description": self.template["IOC-04"]["description"].format(resolved_host, "tracker"),
                                            "host": resolved_host,
                                            "level": "Low",
                                            "id": "IOC-04"})
                    elif host[1] == "doh":
                        if 443 in [p["port"] for p in record["protocols"]]:
                            record["suspicious"] = True
                            self.alerts.append({"title": self.template["IOC-13"]["title"].format(f"{resolved_host}"),
                                                "description": self.template["IOC-13"]["description"].format(f"{resolved_host}"),
                                                "host": resolved_host,
                                                "level": "Low",
                                                "id": "IOC-13"})
                    else:
                        record["suspicious"] = True
                        self.alerts.append({"title": self.template["IOC-01"]["title"].format(resolved_host, record["ip_dst"], host[1].upper()),
                                            "description": self.template["IOC-01"]["description"].format(f"{resolved_host} ({record['ip_dst']})"),
                                            "host": resolved_host,
                                            "level": "High",
                                            "id": "IOC-01"})
                    break
            
            for host in self.tor_nodes:
                if record["ip_dst"] == host:
                    record["suspicious"] = True
                    self.alerts.append({"title": self.template["IOC-11"]["title"].format(resolved_host, record["ip_dst"]),
                                        "description": self.template["IOC-11"]["description"].format(f"{resolved_host} ({record['ip_dst']})"),
                                        "host": resolved_host,
                                        "level": "High",
                                        "id": "IOC-11"})
                    break

            for cidr in self.bl_cidrs:
                try:
                    if IPAddress(record["ip_dst"]) in cidr[0] and any(t in self.indicators_types for t in [cidr[1], "all"]):
                        record["suspicious"] = True
                        self.alerts.append({"title": self.template["IOC-02"]["title"].format(resolved_host, cidr[0], cidr[1].upper()),
                                            "description": self.template["IOC-02"]["description"].format(record["ip_dst"]),
                                            "host": resolved_host,
                                            "level": "Moderate",
                                            "id": "IOC-02"})
                except:
                    continue

        if self.heuristics_analysis:
            for protocol in record["protocols"]:
                if protocol["name"] in ["UDP", "ICMP", "IPV6-ICMP"]:
                    record["suspicious"] = True
                    self.alerts.append({"title": self.template["PROTO-01"]["title"].format(protocol["name"], resolved_host),
                                        "description": self.template["PROTO-01"]["description"].format(protocol["name"], resolved_host),
                                        "host": resolved_host,
                                        "level": "Moderate",
                                        "id": "PROTO-01"})
                try:
                    if protocol["port"] >= int(self.max_ports):
                        record["suspicious"] = True
                        self.alerts.append({"title": self.template["PROTO-02"]["title"].format("", resolved_host,  self.max_ports),
                                            "description": self.template["PROTO-02"]["description"].format("", resolved_host, protocol["port"]),
                                            "host": resolved_host,
                                            "level": "Low",
                                            "id": "PROTO-02"})
                except:
                    pass
                
                if protocol["name"] == "HTTP":
                    record["suspicious"] = True
                    self.alerts.append({"title": self.template["PROTO-03"]["title"].format(resolved_host),
                                        "description": self.template["PROTO-03"]["description"].format(resolved_host),
                                        "host":  resolved_host,
                                        "level": "Low",
                                        "id": "PROTO-03"})

                if protocol["name"] == "HTTP" and protocol["port"] not in self.http_default_ports:
                    record["suspicious"] = True
                    self.alerts.append({"title": self.template["PROTO-04"]["title"].format(resolved_host, protocol["port"]),
                                        "description": self.template["PROTO-04"]["description"].format(resolved_host, protocol["port"]),
                                        "host":  resolved_host,
                                        "level": "Moderate",
                                        "id": "PROTO-04"})

    def check_tls(self, record):
        """Check a TLS protocol and certificates against a set of IOCs / heuristics.
        Note since TLS 1.3, the certificate is not exchanged in clear text, therefore 
        we need to check it "actively" via the method active_check_ssl. 

              1. Check if the TLS record is not using default TLS ports.
              2. Check if one of the certificates is a free one, like Let's Encrypt.
              3. Check if the certificate is auto-signed. 
              4. If the certificate has an SNI, check the domain by calling check_dnsname.
        Args:
            record (dict): record to be processed.
        Returns:
            supicious (bool) : if an alert has been leveraged. 
        """
        if record["whitelisted"]: return

        resolved_host = record["domains"][0] if len(record["domains"]) else record["ip_dst"]
                
        for certificate in record["certificates"]:

            try:
                if "sni" in certificate and certificate["sni"] not in record["domains"]:
                    if certificate["sni"]:
                        if self.check_dnsname(certificate["sni"]):
                            record["suspicious"] = True

                if certificate["port"] not in self.tls_default_ports:
                    record["suspicious"] = True
                    self.alerts.append({"title": self.template["SSL-01"]["title"].format(certificate["port"], resolved_host),
                                        "description": self.template["SSL-01"]["description"].format(resolved_host),
                                        "host": resolved_host,
                                        "level": "Moderate",
                                        "id": "SSL-01"})

                if float(certificate["version"].split(" ")[1]) < 1.3 and "issuerdn" in certificate:

                    if certificate["issuerdn"] in self.free_issuers:
                        record["suspicious"] = True
                        self.alerts.append({"title": self.template["SSL-02"]["title"].format(resolved_host),
                                            "description": self.template["SSL-02"]["description"],
                                            "host": resolved_host,
                                            "level": "Moderate",
                                            "id": "SSL-02"})

                    elif certificate["issuerdn"] == certificate["subject"]:
                        record["suspicious"] = True
                        self.alerts.append({"title": self.template["SSL-03"]["title"].format(resolved_host),
                                            "description": self.template["SSL-03"]["description"].format(resolved_host),
                                            "host": resolved_host,
                                            "level": "Moderate",
                                            "id": "SSL-03"})
                else:
                    if self.active_analysis and self.connected:
                        if "sni" in certificate:
                            if certificate["sni"] not in self.cert_checked:
                                self.cert_checked.append(certificate["sni"])
                                if self.active_check_ssl(certificate["sni"], certificate["port"]):
                                    record["suspicious"] = True
                                    break
                        else:
                            if resolved_host not in self.cert_checked:
                                self.cert_checked.append(resolved_host)
                                if self.active_check_ssl(resolved_host, certificate["port"]):
                                    record["suspicious"] = True
                                    break
            except Exception as e:
                self.errors.append(f"Issue when processing the following certificate (check_tls): {json.dumps(certificate)}")
    
    def get_tor_nodes(self) -> list:
        """Get a list of TOR nodes from dan.me.uk.

        Returns:
            list: list of TOR nodes
        """

        nodes = []
        if os.path.exists("/tmp/tor_nodes.lst"):
            with open("/tmp/tor_nodes.lst", "r") as f:
                for l in f.readlines():
                    nodes.append(l.strip())
        else:
            if self.connected:
                try:
                    nodes_list = requests.get("https://www.dan.me.uk/torlist/", timeout=10).text
                    with open("/tmp/tor_nodes.lst", "w+") as f:
                        f.write(nodes_list)
                    for l in nodes_list.splitlines():
                        nodes.append(l.strip())
                except:
                    self.errors.append(f"Issue when trying to get TOR nodes from dan.me.uk")
        return nodes


    def check_http(self, record):
        """Check the HTTP hostname against a set of IOCs / heuristics.
        Args:
            record (dict): record to be processed.
        Returns:
            supicious (bool) : if an alert has been leveraged. 
        """
        if record["whitelisted"]: return

        if "http" in record:
            for http in record["http"]:
                if http["hostname"] not in record["domains"]:
                    if re.match("^[a-z\.0-9\-]+\.[a-z\-]{2,}$", http["hostname"]):
                        if http["hostname"]:
                            if self.check_dnsname(http["hostname"]):
                                record["suspicious"] = True

    def active_check_ssl(self, host, port):
        """This method:
        
        1. Check the issuer and subject of a certificate directly by connecting
        to the remote server in order to bypass TLS 1.3+ restrictions. 
        Most of this method was been taken from: https://tinyurl.com/3vsvhu79

        2. Get the JARM of the remote server by using the standard poc library
        from sales force. 

        Args:
            host (str): Host to connect to
            port (int): Port to connect to
        """
        try:
            suspect = False
            context = ssl.create_default_context()
            conn = socket.create_connection((host, port))
            sock = context.wrap_socket(conn, server_hostname=host)
            sock.settimeout(5)
            try:
                der_cert = sock.getpeercert(True)
            finally:
                sock.close()

            if "der_cert" in locals():

                certificate = ssl.DER_cert_to_PEM_cert(der_cert)
                x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, certificate)

                issuer = dict(x509.get_issuer().get_components())
                subject = dict(x509.get_subject().get_components())
                certhash = x509.digest("sha1").decode("utf8").replace(":", "").lower()
                issuer = ", ".join(f"{k.decode('utf8')}={v.decode('utf8')}" for k, v in issuer.items())
                subject = ", ".join(f"{k.decode('utf8')}={v.decode('utf8')}" for k, v in subject.items())

                if issuer in self.free_issuers:
                    self.alerts.append({"title": self.template["SSL-02"]["title"].format(host),
                                        "description": self.template["SSL-02"]["description"],
                                        "host": host,
                                        "level": "Moderate",
                                        "id": "SSL-02"})
                    suspect = True

                if issuer == subject:
                    self.alerts.append({"title": self.template["SSL-03"]["title"].format(host),
                                        "description": self.template["SSL-03"]["description"].format(host),
                                        "host": host,
                                        "level": "Moderate",
                                        "id": "SSL-03"})
                    suspect = True
  
                if self.iocs_analysis:
                    for cert in self.bl_certs:
                        if cert[0] == certhash and any(t in self.indicators_types for t in [cert[1], "all"]):
                            self.alerts.append({"title": self.template["SSL-04"]["title"].format(host, cert[1].upper()),
                                                "description": self.template["SSL-04"]["description"].format(host),
                                                "host": host,
                                                "level": "High",
                                                "id": "SSL-04"})
                            suspect = True
                    
                    if self.bl_jarms:
                        host_jarm = get_jarm(host, port)
                        for jarm in self.bl_jarms:
                            if jarm[0] == host_jarm and any(t in self.indicators_types for t in [jarm[1], "all"]):
                                self.alerts.append({"title": self.template["SSL-05"]["title"].format(host, cert[1].upper()),
                                                    "description": self.template["SSL-05"]["description"].format(host),
                                                    "host": host,
                                                    "level": "High",
                                                    "id": "SSL-05"})
                                suspect = True
            return suspect
        except:
            self.errors.append(f"Issue when trying to grab the SSL certificate located at {host}:{port}")
            return False

    def get_alerts(self):
        """Retrieves the alerts triggered during the analysis

        Returns:
            list: list of the alerts.
        """
        self.analysis_end = datetime.now()
        return [dict(t) for t in {tuple(d.items()) for d in self.alerts}]