Add timeout to requests calls

snorkell_ai/config_54b37
main
2024-05-30 21:28:43 +00:00 · 2024-04-14 17:12:47 +00:00 · 2024-04-14 11:59:46 -05:00 · 2024-04-14 11:59:41 -05:00 · 2024-04-14 11:59:11 -05:00 · 2024-04-14 11:52:34 -05:00
22 changed files with 754 additions and 46002 deletions
@@ -0,0 +1,19 @@
+# This workflow will improvise current file with AI genereated documentation and Create new PR
+
+name: Snorkell.ai - Revolutionizing Documentation on GitHub
+
+on:
+  push:
+    branches: ["main"]
+  workflow_dispatch:
+
+jobs:
+  Documentation:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Snorkell DocGen Client
+      uses: SingularityX-ai/snorkell-documentation-client@v1.0.0
+      with:
+        client_id: ${{ secrets.SNORKELL_CLIENT_ID }}
+        api_key: ${{ secrets.SNORKELL_API_KEY }}
+        branch_name: "main"
@@ -1,479 +1,478 @@
-
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from classes.parsezeeklogs import ParseZeekLogs
-from netaddr import IPNetwork, IPAddress
-from utils import get_iocs, get_config, get_whitelist
-from datetime import datetime
-import subprocess as sp
-import json
-import pydig
-import os
-import re
-import sys
-import whois
-from security import safe_command
-
-
-class ZeekEngine(object):
-
-    def __init__(self, capture_directory):
-        self.working_dir = capture_directory
-        self.alerts = []
-        self.conns = []
-        self.ssl = []
-        self.http = []
-        self.dns = []
-        self.files = []
-        self.whitelist = []
-
-        # Get analysis and userlang configuration
-        self.heuristics_analysis = get_config(("analysis", "heuristics"))
-        self.iocs_analysis = get_config(("analysis", "iocs"))
-        self.whitelist_analysis = get_config(("analysis", "whitelist"))
-        self.active_analysis = get_config(("analysis", "active"))
-        self.userlang = get_config(("frontend", "user_lang"))
-
-        # Retreive IOCs.
-        if self.iocs_analysis:
-            self.bl_cidrs = [[IPNetwork(cidr[0]), cidr[1]]
-                             for cidr in get_iocs("cidr")]
-            self.bl_hosts = get_iocs("ip4addr") + get_iocs("ip6addr")
-            self.bl_domains = get_iocs("domain")
-            self.bl_freedns = get_iocs("freedns")
-            self.bl_nameservers = get_iocs("ns")
-            self.bl_tlds = get_iocs("tld")
-
-        # Retreive whitelisted items.
-        if self.whitelist_analysis:
-            self.wl_cidrs = [IPNetwork(cidr) for cidr in get_whitelist("cidr")]
-            self.wl_hosts = get_whitelist("ip4addr") + get_whitelist("ip6addr")
-            self.wl_domains = get_whitelist("domain")
-
-        # Load template language
-        if not re.match("^[a-z]{2,3}$", self.userlang):
-            self.userlang = "en"
-        with open(os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "locales/{}.json".format(self.userlang))) as f:
-            self.template = json.load(f)["alerts"]
-
-    def fill_dns(self, dir):
-        """
-            Fill the DNS resolutions thanks to the dns.log.
-            :return: nothing - all resolutions appended to self.dns.
-        """
-        if os.path.isfile(os.path.join(dir, "dns.log")):
-            for record in ParseZeekLogs(os.path.join(dir, "dns.log"), output_format="json", safe_headers=False):
-                if record is not None:
-                    if record["qtype_name"] in ["A", "AAAA"]:
-                        d = {"domain": record["query"],
-                             "answers": record["answers"].split(",")}
-                        if d not in self.dns:
-                            self.dns.append(d)
-
-    def netflow_check(self, dir):
-        """
-            Enrich and check the netflow from the conn.log against whitelist and IOCs.
-            :return: nothing - all stuff appended to self.alerts
-        """
-        max_ports = get_config(("analysis", "max_ports"))
-        http_default_port = get_config(("analysis", "http_default_port"))
-
-        # Get the netflow from conn.log.
-        if os.path.isfile(os.path.join(dir, "conn.log")):
-            for record in ParseZeekLogs(os.path.join(dir, "conn.log"), output_format="json", safe_headers=False):
-                if record is not None:
-                    c = {"ip_dst": record["id.resp_h"],
-                         "proto": record["proto"],
-                         "port_dst": record["id.resp_p"],
-                         "service": record["service"],
-                         "alert_tiggered": False}
-                    if c not in self.conns:
-                        self.conns.append(c)
-
-        # Let's add some dns resolutions.
-        for c in self.conns:
-            c["resolution"] = self.resolve(c["ip_dst"])
-
-        # Order the conns list by the resolution field.
-        self.conns = sorted(self.conns, key=lambda c: c["resolution"])
-
-        # Check for whitelisted assets, if any, delete the record.
-        if self.whitelist_analysis:
-
-            for i, c in enumerate(self.conns):
-                if c["ip_dst"] in [ip for ip in self.wl_hosts]:
-                    self.whitelist.append(self.conns[i])
-                    self.conns[i] = False
-                elif c["resolution"] in self.wl_domains:
-                    self.whitelist.append(self.conns[i])
-                    self.conns[i] = False
-                elif True in [c["resolution"].endswith("." + dom) for dom in self.wl_domains]:
-                    self.whitelist.append(self.conns[i])
-                    self.conns[i] = False
-                elif True in [IPAddress(c["ip_dst"]) in cidr for cidr in self.wl_cidrs]:
-                    self.whitelist.append(self.conns[i])
-                    self.conns[i] = False
-
-            # Let's delete whitelisted connections.
-            self.conns = list(filter(lambda c: c != False, self.conns))
-
-        if self.heuristics_analysis:
-            for c in self.conns:
-                # Check for UDP / ICMP (strange from a smartphone.)
-                if c["proto"] in ["UDP", "ICMP"]:
-                    c["alert_tiggered"] = True
-                    self.alerts.append({"title": self.template["PROTO-01"]["title"].format(c["proto"].upper(), c["resolution"]),
-                                        "description": self.template["PROTO-01"]["description"].format(c["proto"].upper(), c["resolution"]),
-                                        "host":  c["resolution"],
-                                        "level": "Moderate",
-                                        "id": "PROTO-01"})
-                # Check for use of ports over 1024.
-                if c["port_dst"] >= max_ports:
-                    c["alert_tiggered"] = True
-                    self.alerts.append({"title": self.template["PROTO-02"]["title"].format(c["proto"].upper(), c["resolution"], max_ports),
-                                        "description": self.template["PROTO-02"]["description"].format(c["proto"].upper(), c["resolution"], c["port_dst"]),
-                                        "host":  c["resolution"],
-                                        "level": "Low",
-                                        "id": "PROTO-02"})
-                # Check for use of HTTP.
-                if c["service"] == "http" and c["port_dst"] == http_default_port:
-                    c["alert_tiggered"] = True
-                    self.alerts.append({"title": self.template["PROTO-03"]["title"].format(c["resolution"]),
-                                        "description": self.template["PROTO-03"]["description"].format(c["resolution"]),
-                                        "host":  c["resolution"],
-                                        "level": "Low",
-                                        "id": "PROTO-03"})
-
-                # Check for use of HTTP on a non standard port.
-                if c["service"] == "http" and c["port_dst"] != http_default_port:
-                    c["alert_tiggered"] = True
-                    self.alerts.append({"title": self.template["PROTO-04"]["title"].format(c["resolution"], c["port_dst"]),
-                                        "description": self.template["PROTO-04"]["description"].format(c["resolution"], c["port_dst"]),
-                                        "host":  c["resolution"],
-                                        "level": "Moderate",
-                                        "id": "PROTO-04"})
-                # Check for non-resolved IP address.
-                if c["ip_dst"] == c["resolution"]:
-                    c["alert_tiggered"] = True
-                    self.alerts.append({"title": self.template["PROTO-05"]["title"].format(c["ip_dst"]),
-                                        "description": self.template["PROTO-05"]["description"].format(c["ip_dst"]),
-                                        "host": c["ip_dst"],
-                                        "level": "Low",
-                                        "id": "PROTO-05"})
-
-        if self.iocs_analysis:
-
-            for c in self.conns:
-                # Check for blacklisted IP address.
-                for host in self.bl_hosts:
-                    if c["ip_dst"] == host[0]:
-                        c["alert_tiggered"] = True
-                        self.alerts.append({"title": self.template["IOC-01"]["title"].format(c["resolution"], c["ip_dst"], host[1].upper()),
-                                            "description": self.template["IOC-01"]["description"].format(c["ip_dst"]),
-                                            "host": c["resolution"],
-                                            "level": "High",
-                                            "id": "IOC-01"})
-                        break
-                # Check for blacklisted CIDR.
-                for cidr in self.bl_cidrs:
-                    if IPAddress(c["ip_dst"]) in cidr[0]:
-                        c["alert_tiggered"] = True
-                        self.alerts.append({"title": self.template["IOC-02"]["title"].format(c["resolution"], cidr[0], cidr[1].upper()),
-                                            "description": self.template["IOC-02"]["description"].format(c["resolution"]),
-                                            "host": c["resolution"],
-                                            "level": "Moderate",
-                                            "id": "IOC-02"})
-                # Check for blacklisted domain.
-                for domain in self.bl_domains:
-                    if c["resolution"].endswith(domain[0]):
-                        if domain[1] != "tracker":
-                            c["alert_tiggered"] = True
-                            self.alerts.append({"title": self.template["IOC-03"]["title"].format(c["resolution"], domain[1].upper()),
-                                                "description": self.template["IOC-03"]["description"].format(c["resolution"]),
-                                                "host": c["resolution"],
-                                                "level": "High",
-                                                "id": "IOC-03"})
-                        else:
-                            c["alert_tiggered"] = True
-                            self.alerts.append({"title": self.template["IOC-04"]["title"].format(c["resolution"], domain[1].upper()),
-                                                "description": self.template["IOC-04"]["description"].format(c["resolution"]),
-                                                "host": c["resolution"],
-                                                "level": "Moderate",
-                                                "id": "IOC-04"})
-                # Check for blacklisted FreeDNS.
-                for domain in self.bl_freedns:
-                    if c["resolution"].endswith("." + domain[0]):
-                        c["alert_tiggered"] = True
-                        self.alerts.append({"title": self.template["IOC-05"]["title"].format(c["resolution"]),
-                                            "description": self.template["IOC-05"]["description"].format(c["resolution"]),
-                                            "host": c["resolution"],
-                                            "level": "Moderate",
-                                            "id": "IOC-05"})
-
-                # Check for suspect tlds.
-                for tld in self.bl_tlds:
-                    if c["resolution"].endswith(tld[0]):
-                        c["alert_tiggered"] = True
-                        self.alerts.append({"title": self.template["IOC-06"]["title"].format(c["resolution"]),
-                                            "description": self.template["IOC-06"]["description"].format(c["resolution"], tld[0]),
-                                            "host": c["resolution"],
-                                            "level": "Low",
-                                            "id": "IOC-06"})
-        if self.active_analysis:
-            for c in self.conns:
-                try:  # Domain nameservers check.
-                    name_servers = pydig.query(c["resolution"], "NS")
-                    if len(name_servers):
-                        for ns in self.bl_nameservers:
-                            if name_servers[0].endswith(".{}.".format(ns[0])):
-                                c["alert_tiggered"] = True
-                                self.alerts.append({"title": self.template["ACT-01"]["title"].format(c["resolution"], name_servers[0]),
-                                                    "description": self.template["ACT-01"]["description"].format(c["resolution"]),
-                                                    "host": c["resolution"],
-                                                    "level": "Moderate",
-                                                    "id": "ACT-01"})
-                except:
-                    pass
-
-                try:  # Domain history check.
-
-                    whois_record = whois.whois(c["resolution"])
-                    creation_date = whois_record.creation_date if type(
-                        whois_record.creation_date) is not list else whois_record.creation_date[0]
-                    creation_days = abs((datetime.now() - creation_date).days)
-                    if creation_days < 365:
-                        c["alert_tiggered"] = True
-                        self.alerts.append({"title": self.template["ACT-02"]["title"].format(c["resolution"], creation_days),
-                                            "description": self.template["ACT-02"]["description"].format(c["resolution"]),
-                                            "host": c["resolution"],
-                                            "level": "Moderate",
-                                            "id": "ACT-02"})
-
-                except:
-                    pass
-
-    def files_check(self, dir):
-        """
-            Check on the files.log:
-                * Check certificates sha1
-                * [todo] Check possible binary data or APKs?
-            :return: nothing - all stuff appended to self.alerts
-        """
-
-        if not self.iocs_analysis:
-            return
-
-        bl_certs = get_iocs("sha1cert")
-
-        if os.path.isfile(os.path.join(dir, "files.log")):
-            for record in ParseZeekLogs(os.path.join(dir, "files.log"), output_format="json", safe_headers=False):
-                if record is not None:
-                    f = {"filename": record["filename"],
-                         "ip_src": record["id.orig_h"],
-                         "ip_dst": record["id.resp_h"],
-                         "mime_type": record["mime_type"],
-                         "sha1": record["sha1"]}
-                    if f not in self.files:
-                        self.files.append(f)
-
-        for f in self.files:
-            if f["mime_type"] == "application/x-x509-user-cert":
-                for cert in bl_certs:  # Check for blacklisted certificate.
-                    if f["sha1"] == cert[0]:
-                        host = self.resolve(f["ip_src"])
-                        self.alerts.append({"title": self.template["IOC-07"]["title"].format(cert[1].upper(), host),
-                                            "description": self.template["IOC-07"]["description"].format(f["sha1"], host),
-                                            "host": host,
-                                            "level": "High",
-                                            "id": "IOC-07"})
-
-    def http_check(self, dir):
-        """
-            Check on the http.log:
-                * Blacklisted domain/tld from the Host header field.
-            Can be used when no DNS query have been done during the session (already cached by the device.)
-            :return: nothing - all stuff appended to self.alerts
-        """
-
-        if os.path.isfile(os.path.join(dir, "http.log")):
-            for record in ParseZeekLogs(os.path.join(dir, "http.log"), output_format="json", safe_headers=False):
-                if record is not None:
-                    c = {"host": record['host']}
-                    if c not in self.http:
-                        self.http.append(c)
-
-        if self.iocs_analysis:
-            for c in self.http:
-
-                # If we already know the host form DNS, let's continue.
-                if c["host"] in [r["domain"] for r in self.dns]:
-                    continue
-
-                # Check for blacklisted domain.
-                for h in self.bl_domains:
-                    if h[1] != "tracker":
-                        if c["host"].endswith(h[0]):
-                            self.alerts.append({"title": self.template["IOC-08"]["title"].format(c["host"], h[1].upper()),
-                                                "description": self.template["IOC-08"]["description"].format(c["host"]),
-                                                "host": c["host"],
-                                                "level": "High",
-                                                "id": "IOC-08"})
-                # Check for freedns.
-                for h in self.bl_freedns:
-                    if c["host"].endswith(h[0]):
-                        self.alerts.append({"title": self.template["IOC-09"]["title"].format(c["host"]),
-                                            "description": self.template["IOC-09"]["description"].format(c["host"]),
-                                            "host": c["host"],
-                                            "level": "Moderate",
-                                            "id": "IOC-09"})
-                # Check for fancy TLD.
-                for h in self.bl_tlds:
-                    if c["host"].endswith(h[0]):
-                        self.alerts.append({"title": self.template["IOC-10"]["title"].format(c["host"]),
-                                            "description": self.template["IOC-10"]["description"].format(c["host"], h[0]),
-                                            "host": c["host"],
-                                            "level": "Low",
-                                            "id": "IOC-10"})
-
-    def ssl_check(self, dir):
-        """
-            Check on the ssl.log:
-                * SSL connections which doesn't use the 443.
-                * "Free" certificate issuer (taken from the config).
-                * Self-signed certificates.
-                * Blacklisted domain in the CN
-            :return: nothing - all stuff appended to self.alerts
-        """
-        ssl_default_ports = get_config(("analysis", "ssl_default_ports"))
-        free_issuers = get_config(("analysis", "free_issuers"))
-
-        if os.path.isfile(os.path.join(dir, "ssl.log")):
-            for record in ParseZeekLogs(os.path.join(dir, "ssl.log"), output_format="json", safe_headers=False):
-                if record is not None:
-                    c = {"host": record['id.resp_h'],
-                         "port": record['id.resp_p'],
-                         "issuer": record["issuer"] if "issuer" in record else "",
-                         "validation_status": record["validation_status"],
-                         "cn": record["server_name"] if "server_name" in record else ""}
-                    if c not in self.ssl:
-                        self.ssl.append(c)
-
-        if self.heuristics_analysis:
-            for cert in self.ssl:
-                host = self.resolve(cert["host"])
-
-                # If the associated host has not whitelisted, check the cert.
-                for c in self.conns:
-                    if host in c["resolution"]:
-                        # Check for non generic SSL port.
-                        if cert["port"] not in ssl_default_ports:
-                            c["alert_tiggered"] = True
-                            self.alerts.append({"title": self.template["SSL-01"]["title"].format(cert["port"], host),
-                                                "description": self.template["SSL-01"]["description"].format(host),
-                                                "host": host,
-                                                "level": "Moderate",
-                                                "id": "SSL-01"})
-                        # Check Free SSL certificates.
-                        if cert["issuer"] in free_issuers:
-                            c["alert_tiggered"] = True
-                            self.alerts.append({"title": self.template["SSL-02"]["title"].format(host),
-                                                "description": self.template["SSL-02"]["description"],
-                                                "host": host,
-                                                "level": "Moderate",
-                                                "id": "SSL-02"})
-                        # Check for self-signed certificates.
-                        if cert["validation_status"] == "self signed certificate in certificate chain":
-                            c["alert_tiggered"] = True
-                            self.alerts.append({"title": self.template["SSL-03"]["title"].format(host),
-                                                "description": self.template["SSL-03"]["description"].format(host),
-                                                "host": host,
-                                                "level": "Moderate",
-                                                "id": "SSL-03"})
-
-        if self.iocs_analysis:
-            for cert in self.ssl:
-                # Check if the domain in the certificate haven't been blacklisted
-                # This check can be good if the domain has already been cached by
-                # the device so it wont appear in self.dns.
-
-                if any(cert["cn"].endswith(r["domain"]) for r in self.dns):
-                    continue
-
-                for domain in self.bl_domains:
-                    if domain[1] != "tracker":
-                        if cert["cn"].endswith(domain[0]):
-                            self.alerts.append({"title": self.template["SSL-04"]["title"].format(domain[0], domain[1].upper()),
-                                                "description": self.template["SSL-04"]["description"].format(domain[0]),
-                                                "host": domain[0],
-                                                "level": "High",
-                                                "id": "SSL-04"})
-
-    def alerts_check(self):
-        """
-            Leverage an advice to the user based on the trigered hosts
-            :return: nothing - all generated alerts appended to self.alerts
-        """
-        hosts = {}
-
-        for alert in [dict(t) for t in {tuple(d.items()) for d in self.alerts}]:
-            if alert["host"] not in hosts:
-                hosts[alert["host"]] = 1
-            else:
-                hosts[alert["host"]] += 1
-
-        for host, nb in hosts.items():
-            if nb >= get_config(("analysis", "max_alerts")):
-                self.alerts.append({"title": self.template["ADV-01"]["title"].format(host),
-                                    "description": self.template["ADV-01"]["description"].format(host, nb),
-                                    "host": host,
-                                    "level": "Moderate",
-                                    "id": "ADV-01"})
-
-    def resolve(self, ip_addr):
-        """
-            A simple method to retreive DNS names from IP addresses
-            in order to replace them in alerts.
-
-            :return: String - DNS record or IP Address.
-        """
-        for record in self.dns:
-            if ip_addr in record["answers"]:
-                return record["domain"]
-        return ip_addr
-
-    def start_zeek(self):
-        """
-            Start zeek and check the logs.
-        """
-        safe_command.run(sp.Popen, "cd {} && /opt/zeek/bin/zeek -Cr capture.pcap protocols/ssl/validate-certs".format(
-            self.working_dir), shell=False).wait()
-        safe_command.run(sp.Popen, "cd {} && mv *.log assets/".format(self.working_dir),
-                 shell=False).wait()
-        self.fill_dns(self.working_dir + "/assets/")
-        self.netflow_check(self.working_dir + "/assets/")
-        self.ssl_check(self.working_dir + "/assets/")
-        self.http_check(self.working_dir + "/assets/")
-        self.files_check(self.working_dir + "/assets/")
-        self.alerts_check()
-
-    def retrieve_alerts(self):
-        """
-            Retrieve alerts.
-            :return: list - a list of alerts wihout duplicates.
-        """
-        return [dict(t) for t in {tuple(d.items()) for d in self.alerts}]
-
-    def retrieve_whitelist(self):
-        """
-            Retrieve whitelisted elements.
-            :return: list - a list of whitelisted elements wihout duplicates.
-        """
-        return [dict(t) for t in {tuple(d.items()) for d in self.whitelist}]
-
-    def retrieve_conns(self):
-        """
-            Retrieve not whitelisted elements.
-            :return: list - a list of non-whitelisted elements wihout duplicates.
-        """
-        return self.conns
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from classes.parsezeeklogs import ParseZeekLogs
+from netaddr import IPNetwork, IPAddress
+from utils import get_iocs, get_config, get_whitelist
+from datetime import datetime
+import subprocess as sp
+import json
+import pydig
+import os
+import re
+import sys
+import whois
+
+
+class ZeekEngine(object):
+
+    def __init__(self, capture_directory):
+        self.working_dir = capture_directory
+        self.alerts = []
+        self.conns = []
+        self.ssl = []
+        self.http = []
+        self.dns = []
+        self.files = []
+        self.whitelist = []
+
+        # Get analysis and userlang configuration
+        self.heuristics_analysis = get_config(("analysis", "heuristics"))
+        self.iocs_analysis = get_config(("analysis", "iocs"))
+        self.whitelist_analysis = get_config(("analysis", "whitelist"))
+        self.active_analysis = get_config(("analysis", "active"))
+        self.userlang = get_config(("frontend", "user_lang"))
+
+        # Retreive IOCs.
+        if self.iocs_analysis:
+            self.bl_cidrs = [[IPNetwork(cidr[0]), cidr[1]]
+                             for cidr in get_iocs("cidr")]
+            self.bl_hosts = get_iocs("ip4addr") + get_iocs("ip6addr")
+            self.bl_domains = get_iocs("domain")
+            self.bl_freedns = get_iocs("freedns")
+            self.bl_nameservers = get_iocs("ns")
+            self.bl_tlds = get_iocs("tld")
+
+        # Retreive whitelisted items.
+        if self.whitelist_analysis:
+            self.wl_cidrs = [IPNetwork(cidr) for cidr in get_whitelist("cidr")]
+            self.wl_hosts = get_whitelist("ip4addr") + get_whitelist("ip6addr")
+            self.wl_domains = get_whitelist("domain")
+
+        # Load template language
+        if not re.match("^[a-z]{2,3}$", self.userlang):
+            self.userlang = "en"
+        with open(os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "locales/{}.json".format(self.userlang))) as f:
+            self.template = json.load(f)["alerts"]
+
+    def fill_dns(self, dir):
+        """
+            Fill the DNS resolutions thanks to the dns.log.
+            :return: nothing - all resolutions appended to self.dns.
+        """
+        if os.path.isfile(os.path.join(dir, "dns.log")):
+            for record in ParseZeekLogs(os.path.join(dir, "dns.log"), output_format="json", safe_headers=False):
+                if record is not None:
+                    if record["qtype_name"] in ["A", "AAAA"]:
+                        d = {"domain": record["query"],
+                             "answers": record["answers"].split(",")}
+                        if d not in self.dns:
+                            self.dns.append(d)
+
+    def netflow_check(self, dir):
+        """
+            Enrich and check the netflow from the conn.log against whitelist and IOCs.
+            :return: nothing - all stuff appended to self.alerts
+        """
+        max_ports = get_config(("analysis", "max_ports"))
+        http_default_port = get_config(("analysis", "http_default_port"))
+
+        # Get the netflow from conn.log.
+        if os.path.isfile(os.path.join(dir, "conn.log")):
+            for record in ParseZeekLogs(os.path.join(dir, "conn.log"), output_format="json", safe_headers=False):
+                if record is not None:
+                    c = {"ip_dst": record["id.resp_h"],
+                         "proto": record["proto"],
+                         "port_dst": record["id.resp_p"],
+                         "service": record["service"],
+                         "alert_tiggered": False}
+                    if c not in self.conns:
+                        self.conns.append(c)
+
+        # Let's add some dns resolutions.
+        for c in self.conns:
+            c["resolution"] = self.resolve(c["ip_dst"])
+
+        # Order the conns list by the resolution field.
+        self.conns = sorted(self.conns, key=lambda c: c["resolution"])
+
+        # Check for whitelisted assets, if any, delete the record.
+        if self.whitelist_analysis:
+
+            for i, c in enumerate(self.conns):
+                if c["ip_dst"] in [ip for ip in self.wl_hosts]:
+                    self.whitelist.append(self.conns[i])
+                    self.conns[i] = False
+                elif c["resolution"] in self.wl_domains:
+                    self.whitelist.append(self.conns[i])
+                    self.conns[i] = False
+                elif True in [c["resolution"].endswith("." + dom) for dom in self.wl_domains]:
+                    self.whitelist.append(self.conns[i])
+                    self.conns[i] = False
+                elif True in [IPAddress(c["ip_dst"]) in cidr for cidr in self.wl_cidrs]:
+                    self.whitelist.append(self.conns[i])
+                    self.conns[i] = False
+
+            # Let's delete whitelisted connections.
+            self.conns = list(filter(lambda c: c != False, self.conns))
+
+        if self.heuristics_analysis:
+            for c in self.conns:
+                # Check for UDP / ICMP (strange from a smartphone.)
+                if c["proto"] in ["UDP", "ICMP"]:
+                    c["alert_tiggered"] = True
+                    self.alerts.append({"title": self.template["PROTO-01"]["title"].format(c["proto"].upper(), c["resolution"]),
+                                        "description": self.template["PROTO-01"]["description"].format(c["proto"].upper(), c["resolution"]),
+                                        "host":  c["resolution"],
+                                        "level": "Moderate",
+                                        "id": "PROTO-01"})
+                # Check for use of ports over 1024.
+                if c["port_dst"] >= max_ports:
+                    c["alert_tiggered"] = True
+                    self.alerts.append({"title": self.template["PROTO-02"]["title"].format(c["proto"].upper(), c["resolution"], max_ports),
+                                        "description": self.template["PROTO-02"]["description"].format(c["proto"].upper(), c["resolution"], c["port_dst"]),
+                                        "host":  c["resolution"],
+                                        "level": "Low",
+                                        "id": "PROTO-02"})
+                # Check for use of HTTP.
+                if c["service"] == "http" and c["port_dst"] == http_default_port:
+                    c["alert_tiggered"] = True
+                    self.alerts.append({"title": self.template["PROTO-03"]["title"].format(c["resolution"]),
+                                        "description": self.template["PROTO-03"]["description"].format(c["resolution"]),
+                                        "host":  c["resolution"],
+                                        "level": "Low",
+                                        "id": "PROTO-03"})
+
+                # Check for use of HTTP on a non standard port.
+                if c["service"] == "http" and c["port_dst"] != http_default_port:
+                    c["alert_tiggered"] = True
+                    self.alerts.append({"title": self.template["PROTO-04"]["title"].format(c["resolution"], c["port_dst"]),
+                                        "description": self.template["PROTO-04"]["description"].format(c["resolution"], c["port_dst"]),
+                                        "host":  c["resolution"],
+                                        "level": "Moderate",
+                                        "id": "PROTO-04"})
+                # Check for non-resolved IP address.
+                if c["ip_dst"] == c["resolution"]:
+                    c["alert_tiggered"] = True
+                    self.alerts.append({"title": self.template["PROTO-05"]["title"].format(c["ip_dst"]),
+                                        "description": self.template["PROTO-05"]["description"].format(c["ip_dst"]),
+                                        "host": c["ip_dst"],
+                                        "level": "Low",
+                                        "id": "PROTO-05"})
+
+        if self.iocs_analysis:
+
+            for c in self.conns:
+                # Check for blacklisted IP address.
+                for host in self.bl_hosts:
+                    if c["ip_dst"] == host[0]:
+                        c["alert_tiggered"] = True
+                        self.alerts.append({"title": self.template["IOC-01"]["title"].format(c["resolution"], c["ip_dst"], host[1].upper()),
+                                            "description": self.template["IOC-01"]["description"].format(c["ip_dst"]),
+                                            "host": c["resolution"],
+                                            "level": "High",
+                                            "id": "IOC-01"})
+                        break
+                # Check for blacklisted CIDR.
+                for cidr in self.bl_cidrs:
+                    if IPAddress(c["ip_dst"]) in cidr[0]:
+                        c["alert_tiggered"] = True
+                        self.alerts.append({"title": self.template["IOC-02"]["title"].format(c["resolution"], cidr[0], cidr[1].upper()),
+                                            "description": self.template["IOC-02"]["description"].format(c["resolution"]),
+                                            "host": c["resolution"],
+                                            "level": "Moderate",
+                                            "id": "IOC-02"})
+                # Check for blacklisted domain.
+                for domain in self.bl_domains:
+                    if c["resolution"].endswith(domain[0]):
+                        if domain[1] != "tracker":
+                            c["alert_tiggered"] = True
+                            self.alerts.append({"title": self.template["IOC-03"]["title"].format(c["resolution"], domain[1].upper()),
+                                                "description": self.template["IOC-03"]["description"].format(c["resolution"]),
+                                                "host": c["resolution"],
+                                                "level": "High",
+                                                "id": "IOC-03"})
+                        else:
+                            c["alert_tiggered"] = True
+                            self.alerts.append({"title": self.template["IOC-04"]["title"].format(c["resolution"], domain[1].upper()),
+                                                "description": self.template["IOC-04"]["description"].format(c["resolution"]),
+                                                "host": c["resolution"],
+                                                "level": "Moderate",
+                                                "id": "IOC-04"})
+                # Check for blacklisted FreeDNS.
+                for domain in self.bl_freedns:
+                    if c["resolution"].endswith("." + domain[0]):
+                        c["alert_tiggered"] = True
+                        self.alerts.append({"title": self.template["IOC-05"]["title"].format(c["resolution"]),
+                                            "description": self.template["IOC-05"]["description"].format(c["resolution"]),
+                                            "host": c["resolution"],
+                                            "level": "Moderate",
+                                            "id": "IOC-05"})
+
+                # Check for suspect tlds.
+                for tld in self.bl_tlds:
+                    if c["resolution"].endswith(tld[0]):
+                        c["alert_tiggered"] = True
+                        self.alerts.append({"title": self.template["IOC-06"]["title"].format(c["resolution"]),
+                                            "description": self.template["IOC-06"]["description"].format(c["resolution"], tld[0]),
+                                            "host": c["resolution"],
+                                            "level": "Low",
+                                            "id": "IOC-06"})
+        if self.active_analysis:
+            for c in self.conns:
+                try:  # Domain nameservers check.
+                    name_servers = pydig.query(c["resolution"], "NS")
+                    if len(name_servers):
+                        for ns in self.bl_nameservers:
+                            if name_servers[0].endswith(".{}.".format(ns[0])):
+                                c["alert_tiggered"] = True
+                                self.alerts.append({"title": self.template["ACT-01"]["title"].format(c["resolution"], name_servers[0]),
+                                                    "description": self.template["ACT-01"]["description"].format(c["resolution"]),
+                                                    "host": c["resolution"],
+                                                    "level": "Moderate",
+                                                    "id": "ACT-01"})
+                except:
+                    pass
+
+                try:  # Domain history check.
+
+                    whois_record = whois.whois(c["resolution"])
+                    creation_date = whois_record.creation_date if type(
+                        whois_record.creation_date) is not list else whois_record.creation_date[0]
+                    creation_days = abs((datetime.now() - creation_date).days)
+                    if creation_days < 365:
+                        c["alert_tiggered"] = True
+                        self.alerts.append({"title": self.template["ACT-02"]["title"].format(c["resolution"], creation_days),
+                                            "description": self.template["ACT-02"]["description"].format(c["resolution"]),
+                                            "host": c["resolution"],
+                                            "level": "Moderate",
+                                            "id": "ACT-02"})
+
+                except:
+                    pass
+
+    def files_check(self, dir):
+        """
+            Check on the files.log:
+                * Check certificates sha1
+                * [todo] Check possible binary data or APKs?
+            :return: nothing - all stuff appended to self.alerts
+        """
+
+        if not self.iocs_analysis:
+            return
+
+        bl_certs = get_iocs("sha1cert")
+
+        if os.path.isfile(os.path.join(dir, "files.log")):
+            for record in ParseZeekLogs(os.path.join(dir, "files.log"), output_format="json", safe_headers=False):
+                if record is not None:
+                    f = {"filename": record["filename"],
+                         "ip_src": record["id.orig_h"],
+                         "ip_dst": record["id.resp_h"],
+                         "mime_type": record["mime_type"],
+                         "sha1": record["sha1"]}
+                    if f not in self.files:
+                        self.files.append(f)
+
+        for f in self.files:
+            if f["mime_type"] == "application/x-x509-user-cert":
+                for cert in bl_certs:  # Check for blacklisted certificate.
+                    if f["sha1"] == cert[0]:
+                        host = self.resolve(f["ip_src"])
+                        self.alerts.append({"title": self.template["IOC-07"]["title"].format(cert[1].upper(), host),
+                                            "description": self.template["IOC-07"]["description"].format(f["sha1"], host),
+                                            "host": host,
+                                            "level": "High",
+                                            "id": "IOC-07"})
+
+    def http_check(self, dir):
+        """
+            Check on the http.log:
+                * Blacklisted domain/tld from the Host header field.
+            Can be used when no DNS query have been done during the session (already cached by the device.)
+            :return: nothing - all stuff appended to self.alerts
+        """
+
+        if os.path.isfile(os.path.join(dir, "http.log")):
+            for record in ParseZeekLogs(os.path.join(dir, "http.log"), output_format="json", safe_headers=False):
+                if record is not None:
+                    c = {"host": record['host']}
+                    if c not in self.http:
+                        self.http.append(c)
+
+        if self.iocs_analysis:
+            for c in self.http:
+
+                # If we already know the host form DNS, let's continue.
+                if c["host"] in [r["domain"] for r in self.dns]:
+                    continue
+
+                # Check for blacklisted domain.
+                for h in self.bl_domains:
+                    if h[1] != "tracker":
+                        if c["host"].endswith(h[0]):
+                            self.alerts.append({"title": self.template["IOC-08"]["title"].format(c["host"], h[1].upper()),
+                                                "description": self.template["IOC-08"]["description"].format(c["host"]),
+                                                "host": c["host"],
+                                                "level": "High",
+                                                "id": "IOC-08"})
+                # Check for freedns.
+                for h in self.bl_freedns:
+                    if c["host"].endswith(h[0]):
+                        self.alerts.append({"title": self.template["IOC-09"]["title"].format(c["host"]),
+                                            "description": self.template["IOC-09"]["description"].format(c["host"]),
+                                            "host": c["host"],
+                                            "level": "Moderate",
+                                            "id": "IOC-09"})
+                # Check for fancy TLD.
+                for h in self.bl_tlds:
+                    if c["host"].endswith(h[0]):
+                        self.alerts.append({"title": self.template["IOC-10"]["title"].format(c["host"]),
+                                            "description": self.template["IOC-10"]["description"].format(c["host"], h[0]),
+                                            "host": c["host"],
+                                            "level": "Low",
+                                            "id": "IOC-10"})
+
+    def ssl_check(self, dir):
+        """
+            Check on the ssl.log:
+                * SSL connections which doesn't use the 443.
+                * "Free" certificate issuer (taken from the config).
+                * Self-signed certificates.
+                * Blacklisted domain in the CN
+            :return: nothing - all stuff appended to self.alerts
+        """
+        ssl_default_ports = get_config(("analysis", "ssl_default_ports"))
+        free_issuers = get_config(("analysis", "free_issuers"))
+
+        if os.path.isfile(os.path.join(dir, "ssl.log")):
+            for record in ParseZeekLogs(os.path.join(dir, "ssl.log"), output_format="json", safe_headers=False):
+                if record is not None:
+                    c = {"host": record['id.resp_h'],
+                         "port": record['id.resp_p'],
+                         "issuer": record["issuer"] if "issuer" in record else "",
+                         "validation_status": record["validation_status"],
+                         "cn": record["server_name"] if "server_name" in record else ""}
+                    if c not in self.ssl:
+                        self.ssl.append(c)
+
+        if self.heuristics_analysis:
+            for cert in self.ssl:
+                host = self.resolve(cert["host"])
+
+                # If the associated host has not whitelisted, check the cert.
+                for c in self.conns:
+                    if host in c["resolution"]:
+                        # Check for non generic SSL port.
+                        if cert["port"] not in ssl_default_ports:
+                            c["alert_tiggered"] = True
+                            self.alerts.append({"title": self.template["SSL-01"]["title"].format(cert["port"], host),
+                                                "description": self.template["SSL-01"]["description"].format(host),
+                                                "host": host,
+                                                "level": "Moderate",
+                                                "id": "SSL-01"})
+                        # Check Free SSL certificates.
+                        if cert["issuer"] in free_issuers:
+                            c["alert_tiggered"] = True
+                            self.alerts.append({"title": self.template["SSL-02"]["title"].format(host),
+                                                "description": self.template["SSL-02"]["description"],
+                                                "host": host,
+                                                "level": "Moderate",
+                                                "id": "SSL-02"})
+                        # Check for self-signed certificates.
+                        if cert["validation_status"] == "self signed certificate in certificate chain":
+                            c["alert_tiggered"] = True
+                            self.alerts.append({"title": self.template["SSL-03"]["title"].format(host),
+                                                "description": self.template["SSL-03"]["description"].format(host),
+                                                "host": host,
+                                                "level": "Moderate",
+                                                "id": "SSL-03"})
+
+        if self.iocs_analysis:
+            for cert in self.ssl:
+                # Check if the domain in the certificate haven't been blacklisted
+                # This check can be good if the domain has already been cached by
+                # the device so it wont appear in self.dns.
+
+                if any([cert["cn"].endswith(r["domain"]) for r in self.dns]):
+                    continue
+
+                for domain in self.bl_domains:
+                    if domain[1] != "tracker":
+                        if cert["cn"].endswith(domain[0]):
+                            self.alerts.append({"title": self.template["SSL-04"]["title"].format(domain[0], domain[1].upper()),
+                                                "description": self.template["SSL-04"]["description"].format(domain[0]),
+                                                "host": domain[0],
+                                                "level": "High",
+                                                "id": "SSL-04"})
+
+    def alerts_check(self):
+        """
+            Leverage an advice to the user based on the trigered hosts
+            :return: nothing - all generated alerts appended to self.alerts
+        """
+        hosts = {}
+
+        for alert in [dict(t) for t in {tuple(d.items()) for d in self.alerts}]:
+            if alert["host"] not in hosts:
+                hosts[alert["host"]] = 1
+            else:
+                hosts[alert["host"]] += 1
+
+        for host, nb in hosts.items():
+            if nb >= get_config(("analysis", "max_alerts")):
+                self.alerts.append({"title": self.template["ADV-01"]["title"].format(host),
+                                    "description": self.template["ADV-01"]["description"].format(host, nb),
+                                    "host": host,
+                                    "level": "Moderate",
+                                    "id": "ADV-01"})
+
+    def resolve(self, ip_addr):
+        """
+            A simple method to retreive DNS names from IP addresses
+            in order to replace them in alerts.
+
+            :return: String - DNS record or IP Address.
+        """
+        for record in self.dns:
+            if ip_addr in record["answers"]:
+                return record["domain"]
+        return ip_addr
+
+    def start_zeek(self):
+        """
+            Start zeek and check the logs.
+        """
+        sp.Popen("cd {} && /opt/zeek/bin/zeek -Cr capture.pcap protocols/ssl/validate-certs".format(
+            self.working_dir), shell=True).wait()
+        sp.Popen("cd {} && mv *.log assets/".format(self.working_dir),
+                 shell=True).wait()
+        self.fill_dns(self.working_dir + "/assets/")
+        self.netflow_check(self.working_dir + "/assets/")
+        self.ssl_check(self.working_dir + "/assets/")
+        self.http_check(self.working_dir + "/assets/")
+        self.files_check(self.working_dir + "/assets/")
+        self.alerts_check()
+
+    def retrieve_alerts(self):
+        """
+            Retrieve alerts.
+            :return: list - a list of alerts wihout duplicates.
+        """
+        return [dict(t) for t in {tuple(d.items()) for d in self.alerts}]
+
+    def retrieve_whitelist(self):
+        """
+            Retrieve whitelisted elements.
+            :return: list - a list of whitelisted elements wihout duplicates.
+        """
+        return [dict(t) for t in {tuple(d.items()) for d in self.whitelist}]
+
+    def retrieve_conns(self):
+        """
+            Retrieve not whitelisted elements.
+            :return: list - a list of non-whitelisted elements wihout duplicates.
+        """
+        return self.conns
@@ -1,4 +1 @@
-<svg width="112" height="195" viewBox="0 0 112 195" fill="none" xmlns="http://www.w3.org/2000/svg">
-<line x1="3.5" y1="3.5" x2="3.50001" y2="191.5" stroke="black" stroke-width="7" stroke-linecap="round" stroke-linejoin="round"/>
-<rect x="7" width="105" height="195" fill="#F7F8F9"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="112" height="195" fill="none" viewBox="0 0 112 195"><line x1="3.5" x2="3.5" y1="3.5" y2="191.5" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="7"/><rect width="105" height="195" x="7" fill="#F7F8F9"/></svg>
@@ -1,16 +1 @@
-<svg version="1.1" id="loader-1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-   width="40px" height="40px" viewBox="0 0 40 40" enable-background="new 0 0 40 40" xml:space="preserve">
-  <path opacity="0.2" fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946
-    s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634
-    c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z"/>
-  <path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0
-    C22.32,8.481,24.301,9.057,26.013,10.047z">
-    <animateTransform attributeType="xml"
-      attributeName="transform"
-      type="rotate"
-      from="0 20 20"
-      to="360 20 20"
-      dur="0.5s"
-      repeatCount="indefinite"/>
-    </path>
-  </svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="loader-1" width="40" height="40" x="0" y="0" enable-background="new 0 0 40 40" version="1.1" viewBox="0 0 40 40" xml:space="preserve"><path fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946 s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634 c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z" opacity=".2"/><path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0 C22.32,8.481,24.301,9.057,26.013,10.047z"><animateTransform attributeName="transform" attributeType="xml" dur="0.5s" from="0 20 20" repeatCount="indefinite" to="360 20 20" type="rotate"/></path></svg>
@@ -1,5 +1 @@
-<svg width="106" height="106" viewBox="0 0 106 106" fill="none" xmlns="http://www.w3.org/2000/svg">
-<circle cx="53" cy="53" r="53" fill="#40D8A1"/>
-<path d="M29 52.5L47.5 70.5" stroke="white" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"/>
-<line x1="79" y1="40.0711" x2="48.0711" y2="71" stroke="white" stroke-width="10" stroke-linecap="round"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="106" height="106" fill="none" viewBox="0 0 106 106"><circle cx="53" cy="53" r="53" fill="#40D8A1"/><path stroke="#fff" stroke-linecap="round" stroke-linejoin="round" stroke-width="10" d="M29 52.5L47.5 70.5"/><line x1="79" x2="48.071" y1="40.071" y2="71" stroke="#fff" stroke-linecap="round" stroke-width="10"/></svg>
@@ -1,6 +1 @@
-<svg width="548" height="199" viewBox="0 0 548 199" fill="none" xmlns="http://www.w3.org/2000/svg">
-<rect x="403" y="27" width="142" height="145" rx="8" fill="white" stroke="black" stroke-width="6" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z" fill="black"/>
-<rect x="477" y="55" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
-<rect x="477" y="117" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="548" height="199" fill="none" viewBox="0 0 548 199"><rect width="142" height="145" x="403" y="27" fill="#fff" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="6" rx="8"/><path fill="#000" d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z"/><rect width="26" height="26" x="477" y="55" fill="#fff" stroke="#000" stroke-width="6"/><rect width="26" height="26" x="477" y="117" fill="#fff" stroke="#000" stroke-width="6"/></svg>
@@ -1,11 +1 @@
-<?xml version="1.0" encoding="utf-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin: auto; background: none; display: block; shape-rendering: auto;" width="200px" height="200px" viewBox="0 0 100 100" preserveAspectRatio="xMidYMid">
-<circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2">
-  <animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
-  <animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
-</circle>
-<circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2">
-  <animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline"></animate>
-  <animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline"></animate>
-</circle>
-<!-- [ldio] generated by https://loading.io/ --></svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin:auto;background:0 0;display:block;shape-rendering:auto" width="200" height="200" preserveAspectRatio="xMidYMid" viewBox="0 0 100 100"><circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2"><animate attributeName="r" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle><circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2"><animate attributeName="r" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle></svg>
@@ -1,20 +1,17 @@
-pymisp==2.4.165.1
-sqlalchemy==1.4.48
-ipwhois==1.2.0
-netaddr==0.8.0
-flask==1.1.2
-flask_httpauth==4.8.0
-pyjwt==1.7.1
-psutil==5.8.0
-pydig==0.4.0
-pyudev==0.24.0
-pyyaml==5.3.1
-wifi==0.3.8
-qrcode==7.3.1
-netifaces==0.11.0
-weasyprint==59.0
-python-whois==0.8.0
-six==1.16.0
-security==1.2.1 \
-    --hash=sha256:4ca5f8cfc6b836e2192a84bb5a28b72c17f3cd1abbfe3281f917394c6e6c9238
-    --hash=sha256:0a9dc7b457330e6d0f92bdae3603fecb85394beefad0fd3b5058758a58781ded
+pymisp==2.4.165.1
+sqlalchemy==1.4.48
+ipwhois==1.2.0
+netaddr==0.8.0
+flask==2.2.5
+flask_httpauth==4.8.0
+pyjwt==2.4.0
+psutil==5.8.0
+pydig==0.4.0
+pyudev==0.24.0
+pyyaml==5.3.1
+wifi==0.3.8
+qrcode==7.3.1
+netifaces==0.11.0
+weasyprint==59.0
+python-whois==0.8.0
+six==1.16.0
@@ -1,149 +1,149 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from app.utils import read_config
-from app.classes.iocs import IOCs
-from app.classes.whitelist import WhiteList
-from app.classes.misp import MISP
-
-import requests
-import json
-import urllib3
-import time
-from multiprocessing import Process
-
-"""
-    This file is parsing the watchers present
-    in the configuration file. This in order to get
-    automatically new iocs / elements from remote
-    sources without user interaction.
-"""
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-
-def watch_iocs():
-    """
-        Retrieve IOCs from the remote URLs defined in config/watchers.
-        For each IOC, add it to the DB.
-    """
-
-    # Retrieve the URLs from the configuration
-    urls = read_config(("watchers", "iocs"))
-    watchers = [{"url": url, "status": False} for url in urls]
-
-    while True:
-        for w in watchers:
-            if w["status"] == False:
-                iocs = IOCs()
-                iocs_list = []
-                to_delete = []
-                try:
-                    res = requests.get(w["url"], verify=False)
-                    if res.status_code == 200:
-                        content = json.loads(res.content)
-                        iocs_list = content["iocs"] if "iocs" in content else []
-                        to_delete = content["to_delete"] if "to_delete" in content else []
-                    else:
-                        w["status"] = False
-                except:
-                    w["status"] = False
-
-                for ioc in iocs_list:
-                    try:
-                        iocs.add(ioc["type"], ioc["tag"],
-                                 ioc["tlp"], ioc["value"], "watcher")
-                        w["status"] = True
-                    except:
-                        continue
-
-                for ioc in to_delete:
-                    try:
-                        iocs.delete_by_value(ioc["value"])
-                        w["status"] = True
-                    except:
-                        continue
-
-        # If at least one URL haven't be parsed, let's retry in 1min.
-        if False in [w["status"] for w in watchers]:
-            time.sleep(60)
-        else:
-            break
-
-
-def watch_whitelists():
-    """
-        Retrieve whitelist elements from the remote URLs
-        defined in config/watchers. For each (new ?) element,
-        add it to the DB.
-    """
-
-    urls = read_config(("watchers", "whitelists"))
-    watchers = [{"url": url, "status": False} for url in urls]
-
-    while True:
-        for w in watchers:
-            if w["status"] == False:
-                whitelist = WhiteList()
-                elements = []
-                to_delete = []
-                try:
-                    res = requests.get(w["url"], verify=False)
-                    if res.status_code == 200:
-                        content = json.loads(res.content)
-                        elements = content["elements"] if "elements" in content else []
-                        to_delete = content["to_delete"] if "to_delete" in content else []
-                    else:
-                        w["status"] = False
-                except:
-                    w["status"] = False
-
-                for elem in elements:
-                    try:
-                        whitelist.add(elem["type"], elem["element"], "watcher")
-                        w["status"] = True
-                    except:
-                        continue
-
-                for elem in to_delete:
-                    try:
-                        whitelist.delete_by_value(elem["element"])
-                        w["status"] = True
-                    except:
-                        continue
-
-        if False in [w["status"] for w in watchers]:
-            time.sleep(60)
-        else:
-            break
-
-
-def watch_misp():
-    """
-        Retrieve IOCs from misp instances. Each new element is
-        tested and then added to the database.
-    """
-    iocs, misp = IOCs(), MISP()
-    instances = [i for i in misp.get_instances()]
-
-    while instances:
-        for i, ist in enumerate(instances):
-            status = misp.test_instance(ist["url"],
-                                        ist["apikey"],
-                                        ist["verifycert"])
-            if status:
-                for ioc in misp.get_iocs(ist["id"]):
-                    iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
-                             ioc["value"], "misp-{}".format(ist["id"]))
-                misp.update_sync(ist["id"])
-                instances.pop(i)
-        if instances: time.sleep(60)
-
-
-p1 = Process(target=watch_iocs)
-p2 = Process(target=watch_whitelists)
-p3 = Process(target=watch_misp)
-
-p1.start()
-p2.start()
-p3.start()
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from app.utils import read_config
+from app.classes.iocs import IOCs
+from app.classes.whitelist import WhiteList
+from app.classes.misp import MISP
+
+import requests
+import json
+import urllib3
+import time
+from multiprocessing import Process
+
+"""
+    This file is parsing the watchers present
+    in the configuration file. This in order to get
+    automatically new iocs / elements from remote
+    sources without user interaction.
+"""
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def watch_iocs():
+    """
+        Retrieve IOCs from the remote URLs defined in config/watchers.
+        For each IOC, add it to the DB.
+    """
+
+    # Retrieve the URLs from the configuration
+    urls = read_config(("watchers", "iocs"))
+    watchers = [{"url": url, "status": False} for url in urls]
+
+    while True:
+        for w in watchers:
+            if w["status"] == False:
+                iocs = IOCs()
+                iocs_list = []
+                to_delete = []
+                try:
+                    res = requests.get(w["url"], verify=False, timeout=60)
+                    if res.status_code == 200:
+                        content = json.loads(res.content)
+                        iocs_list = content["iocs"] if "iocs" in content else []
+                        to_delete = content["to_delete"] if "to_delete" in content else []
+                    else:
+                        w["status"] = False
+                except:
+                    w["status"] = False
+
+                for ioc in iocs_list:
+                    try:
+                        iocs.add(ioc["type"], ioc["tag"],
+                                 ioc["tlp"], ioc["value"], "watcher")
+                        w["status"] = True
+                    except:
+                        continue
+
+                for ioc in to_delete:
+                    try:
+                        iocs.delete_by_value(ioc["value"])
+                        w["status"] = True
+                    except:
+                        continue
+
+        # If at least one URL haven't be parsed, let's retry in 1min.
+        if False in [w["status"] for w in watchers]:
+            time.sleep(60)
+        else:
+            break
+
+
+def watch_whitelists():
+    """
+        Retrieve whitelist elements from the remote URLs
+        defined in config/watchers. For each (new ?) element,
+        add it to the DB.
+    """
+
+    urls = read_config(("watchers", "whitelists"))
+    watchers = [{"url": url, "status": False} for url in urls]
+
+    while True:
+        for w in watchers:
+            if w["status"] == False:
+                whitelist = WhiteList()
+                elements = []
+                to_delete = []
+                try:
+                    res = requests.get(w["url"], verify=False, timeout=60)
+                    if res.status_code == 200:
+                        content = json.loads(res.content)
+                        elements = content["elements"] if "elements" in content else []
+                        to_delete = content["to_delete"] if "to_delete" in content else []
+                    else:
+                        w["status"] = False
+                except:
+                    w["status"] = False
+
+                for elem in elements:
+                    try:
+                        whitelist.add(elem["type"], elem["element"], "watcher")
+                        w["status"] = True
+                    except:
+                        continue
+
+                for elem in to_delete:
+                    try:
+                        whitelist.delete_by_value(elem["element"])
+                        w["status"] = True
+                    except:
+                        continue
+
+        if False in [w["status"] for w in watchers]:
+            time.sleep(60)
+        else:
+            break
+
+
+def watch_misp():
+    """
+        Retrieve IOCs from misp instances. Each new element is
+        tested and then added to the database.
+    """
+    iocs, misp = IOCs(), MISP()
+    instances = [i for i in misp.get_instances()]
+
+    while instances:
+        for i, ist in enumerate(instances):
+            status = misp.test_instance(ist["url"],
+                                        ist["apikey"],
+                                        ist["verifycert"])
+            if status:
+                for ioc in misp.get_iocs(ist["id"]):
+                    iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
+                             ioc["value"], "misp-{}".format(ist["id"]))
+                misp.update_sync(ist["id"])
+                instances.pop(i)
+        if instances: time.sleep(60)
+
+
+p1 = Process(target=watch_iocs)
+p2 = Process(target=watch_whitelists)
+p3 = Process(target=watch_misp)
+
+p1.start()
+p2.start()
+p3.start()
@@ -1,82 +1,82 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from app.utils import read_config
-import subprocess as sp
-import requests
-import json
-import os
-import re
-
-
-class Update(object):
-
-    def __init__(self):
-        self.project_url = "https://api.github.com/repos/KasperskyLab/TinyCheck/tags"
-        self.app_path = "/usr/share/tinycheck"
-        return None
-
-    def check_version(self):
-        """
-            Check if a new version of TinyCheck is available 
-            by quering the Github api and comparing the last
-            tag inside the VERSION file.
-            :return: dict containing the available versions.
-        """
-        if read_config(("frontend", "update")):
-            try:
-                res = requests.get(self.project_url)
-                res = json.loads(res.content.decode("utf8"))
-
-                with open(os.path.join(self.app_path, "VERSION")) as f:
-                    cv = f.read()
-                    if cv != res[0]["name"]:
-                        return {"status": True,
-                                "message": "A new version is available",
-                                "current_version": cv,
-                                "next_version": res[0]["name"]}
-                    else:
-                        return {"status": True,
-                                "message": "This is the latest version",
-                                "current_version": cv}
-            except:
-                return {"status": False,
-                        "message": "Something went wrong (no API access nor version file)"}
-        else:
-            return {"status": False,
-                    "message": "You don't have rights to do this operation."}
-
-    def get_current_version(self):
-        """
-            Get the current version of the TinyCheck instance
-            :return: dict containing the current version or error.
-        """
-        if read_config(("frontend", "update")):
-            try:
-                with open(os.path.join(self.app_path, "VERSION")) as f:
-                    return {"status": True,
-                            "current_version": f.read()}
-            except:
-                return {"status": False,
-                        "message": "Something went wrong - no version file ?"}
-        else:
-            return {"status": False,
-                    "message": "You don't have rights to do this operation."}
-
-    def update_instance(self):
-        """
-            Update the instance by executing the update script.
-            :return: dict containing the update status.
-        """
-        if read_config(("frontend", "update")):
-            try:
-                os.chdir(self.app_path)
-                sp.Popen(["bash", os.path.join(self.app_path, "update.sh")])
-                return {"status": True,
-                        "message": "Update successfully launched"}
-            except:
-                return {"status": False,
-                        "message": "Issue during the update"}
-        else:
-            return {"status": False,
-                    "message": "You don't have rights to do this operation."}
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from app.utils import read_config
+import subprocess as sp
+import requests
+import json
+import os
+import re
+
+
+class Update(object):
+
+    def __init__(self):
+        self.project_url = "https://api.github.com/repos/KasperskyLab/TinyCheck/tags"
+        self.app_path = "/usr/share/tinycheck"
+        return None
+
+    def check_version(self):
+        """
+            Check if a new version of TinyCheck is available 
+            by quering the Github api and comparing the last
+            tag inside the VERSION file.
+            :return: dict containing the available versions.
+        """
+        if read_config(("frontend", "update")):
+            try:
+                res = requests.get(self.project_url, timeout=60)
+                res = json.loads(res.content.decode("utf8"))
+
+                with open(os.path.join(self.app_path, "VERSION")) as f:
+                    cv = f.read()
+                    if cv != res[0]["name"]:
+                        return {"status": True,
+                                "message": "A new version is available",
+                                "current_version": cv,
+                                "next_version": res[0]["name"]}
+                    else:
+                        return {"status": True,
+                                "message": "This is the latest version",
+                                "current_version": cv}
+            except:
+                return {"status": False,
+                        "message": "Something went wrong (no API access nor version file)"}
+        else:
+            return {"status": False,
+                    "message": "You don't have rights to do this operation."}
+
+    def get_current_version(self):
+        """
+            Get the current version of the TinyCheck instance
+            :return: dict containing the current version or error.
+        """
+        if read_config(("frontend", "update")):
+            try:
+                with open(os.path.join(self.app_path, "VERSION")) as f:
+                    return {"status": True,
+                            "current_version": f.read()}
+            except:
+                return {"status": False,
+                        "message": "Something went wrong - no version file ?"}
+        else:
+            return {"status": False,
+                    "message": "You don't have rights to do this operation."}
+
+    def update_instance(self):
+        """
+            Update the instance by executing the update script.
+            :return: dict containing the update status.
+        """
+        if read_config(("frontend", "update")):
+            try:
+                os.chdir(self.app_path)
+                sp.Popen(["bash", os.path.join(self.app_path, "update.sh")])
+                return {"status": True,
+                        "message": "Update successfully launched"}
+            except:
+                return {"status": False,
+                        "message": "Issue during the update"}
+        else:
+            return {"status": False,
+                    "message": "You don't have rights to do this operation."}
Author	SHA1	Message	Date
pixeebot[bot]	0a636e4a76	Add timeout to `requests` calls	2024-05-30 21:28:43 +00:00
snorkell-ai[bot]	3ff9520114	snorkell_ai/config_54b37 main	2024-04-14 17:12:47 +00:00
Robert Bongart (MSc MSc MA)	cf5808cf71	Merge pull request #1 from 2lambda123/dependabot/pip/assets/pip-33fa3f9920 Bump the pip group across 1 directory with 2 updates	2024-04-14 11:59:46 -05:00
Robert Bongart (MSc MSc MA)	052ce4cd9c	Merge pull request #2 from 2lambda123/imgbot [ImgBot] Optimize images	2024-04-14 11:59:41 -05:00
Robert Bongart (MSc MSc MA)	1eecd892f7	Merge pull request #9 from 2lambda123/ablesov/add_diagnostics_script Added diagnostics script	2024-04-14 11:59:11 -05:00
Robert Bongart (MSc MSc MA)	f9be1daccb	Merge pull request #8 from 2lambda123/ablesov/fix_requirements Update requirements.txt	2024-04-14 11:52:34 -05:00
Robert Bongart (MSc MSc MA)	6d0560c311	Merge pull request #6 from 2lambda123/ablesov/pwd_compare_fix Update install.sh	2024-04-14 11:28:44 -05:00
Robert Bongart (MSc MSc MA)	dfdbff5100	Merge pull request #3 from 2lambda123/ablesov/fix_field_name Fixed field mapping @ zeekengine.py	2024-04-14 11:16:37 -05:00
dependabot[bot]	e93a4c16de	Bump the pip group across 1 directory with 2 updates Bumps the pip group with 2 updates in the /assets directory: [flask](https://github.com/pallets/flask) and [pyjwt](https://github.com/jpadilla/pyjwt). Updates `flask` from 1.1.2 to 2.2.5 - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.1.2...2.2.5) Updates `pyjwt` from 1.7.1 to 2.4.0 - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/1.7.1...2.4.0) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production dependency-group: pip - dependency-name: pyjwt dependency-type: direct:production dependency-group: pip ... Signed-off-by: dependabot[bot] <support@github.com>	2024-04-14 16:08:55 +00:00
ImgBotApp	bd2c567151	[ImgBot] Optimize images *Total -- 5,245.48kb -> 4,078.69kb (22.24%) /app/frontend/src/assets/icon.png -- 27.84kb -> 14.24kb (48.84%) /assets/network-home.png -- 315.20kb -> 184.93kb (41.33%) /app/backend/src/assets/network-home.png -- 314.77kb -> 185.00kb (41.23%) /app/backend/src/assets/network.png -- 133.43kb -> 91.76kb (31.22%) /assets/backend.png -- 775.25kb -> 540.26kb (30.31%) /app/frontend/src/assets/logo.png -- 87.56kb -> 63.96kb (26.95%) /assets/frontend.png -- 404.96kb -> 324.34kb (19.91%) /app/frontend/src/assets/fonts/Roboto-Bold.svg -- 805.04kb -> 673.54kb (16.34%) /app/backend/src/assets/fonts/Roboto-Bold.svg -- 805.04kb -> 673.54kb (16.34%) /app/frontend/src/assets/fonts/Roboto-Regular.svg -- 784.41kb -> 660.24kb (15.83%) /app/backend/src/assets/fonts/Roboto-Regular.svg -- 784.41kb -> 660.24kb (15.83%) /app/backend/src/assets/github.png -- 4.17kb -> 3.52kb (15.51%) /app/frontend/src/assets/loading.svg -- 1.21kb -> 1.05kb (13.12%) /app/frontend/src/assets/icon_spinner.svg -- 0.95kb -> 0.85kb (9.9%) /app/frontend/src/assets/icon_plug_usb.svg -- 0.29kb -> 0.27kb (4.42%) /app/frontend/src/assets/icon_success.svg -- 0.37kb -> 0.35kb (3.71%) /app/frontend/src/assets/icon_usb.svg -- 0.59kb -> 0.57kb (3.16%) Signed-off-by: ImgBotApp <ImgBotHelp@gmail.com>	2024-04-14 16:08:12 +00:00
Eugeny Ablesov	4f989dcd94	Fixed field mapping @ zeekengine.py (#143 ) Fixed field mapping @ zeekengine.py: resp_h is destination host, please refer to https://docs.zeek.org/en/current/log-formats.html	2023-08-14 15:14:42 +03:00
Eugeny Ablesov	c3f09469b9	Added diagnostics script	2023-06-26 14:01:58 +03:00
Eugeny Ablesov	69512ba605	Update install.sh Fixed password comparison	2023-06-26 14:00:12 +03:00
Eugeny Ablesov	bd4ab27c21	Update requirements.txt Use only fixed versions for dependencies	2023-06-26 13:57:42 +03:00