Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Robert Bongart (MSc MSc MA)
|
13ddb3b0bc | ||
 pixeebot[bot]
|
daca707ad8 | ||
|
pixeebot[bot]
|
216db5d387 | ||
|
pixeebot[bot]
|
887e90cd06 |
@@ -1,19 +0,0 @@
|
||||
# This workflow will improvise current file with AI genereated documentation and Create new PR
|
||||
|
||||
name: Snorkell.ai - Revolutionizing Documentation on GitHub
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
Documentation:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Snorkell DocGen Client
|
||||
uses: SingularityX-ai/snorkell-documentation-client@v1.0.0
|
||||
with:
|
||||
client_id: ${{ secrets.SNORKELL_CLIENT_ID }}
|
||||
api_key: ${{ secrets.SNORKELL_API_KEY }}
|
||||
branch_name: "main"
|
||||
+479
-478
@@ -1,478 +1,479 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from classes.parsezeeklogs import ParseZeekLogs
|
||||
from netaddr import IPNetwork, IPAddress
|
||||
from utils import get_iocs, get_config, get_whitelist
|
||||
from datetime import datetime
|
||||
import subprocess as sp
|
||||
import json
|
||||
import pydig
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import whois
|
||||
|
||||
|
||||
class ZeekEngine(object):
|
||||
|
||||
def __init__(self, capture_directory):
|
||||
self.working_dir = capture_directory
|
||||
self.alerts = []
|
||||
self.conns = []
|
||||
self.ssl = []
|
||||
self.http = []
|
||||
self.dns = []
|
||||
self.files = []
|
||||
self.whitelist = []
|
||||
|
||||
# Get analysis and userlang configuration
|
||||
self.heuristics_analysis = get_config(("analysis", "heuristics"))
|
||||
self.iocs_analysis = get_config(("analysis", "iocs"))
|
||||
self.whitelist_analysis = get_config(("analysis", "whitelist"))
|
||||
self.active_analysis = get_config(("analysis", "active"))
|
||||
self.userlang = get_config(("frontend", "user_lang"))
|
||||
|
||||
# Retreive IOCs.
|
||||
if self.iocs_analysis:
|
||||
self.bl_cidrs = [[IPNetwork(cidr[0]), cidr[1]]
|
||||
for cidr in get_iocs("cidr")]
|
||||
self.bl_hosts = get_iocs("ip4addr") + get_iocs("ip6addr")
|
||||
self.bl_domains = get_iocs("domain")
|
||||
self.bl_freedns = get_iocs("freedns")
|
||||
self.bl_nameservers = get_iocs("ns")
|
||||
self.bl_tlds = get_iocs("tld")
|
||||
|
||||
# Retreive whitelisted items.
|
||||
if self.whitelist_analysis:
|
||||
self.wl_cidrs = [IPNetwork(cidr) for cidr in get_whitelist("cidr")]
|
||||
self.wl_hosts = get_whitelist("ip4addr") + get_whitelist("ip6addr")
|
||||
self.wl_domains = get_whitelist("domain")
|
||||
|
||||
# Load template language
|
||||
if not re.match("^[a-z]{2,3}$", self.userlang):
|
||||
self.userlang = "en"
|
||||
with open(os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "locales/{}.json".format(self.userlang))) as f:
|
||||
self.template = json.load(f)["alerts"]
|
||||
|
||||
def fill_dns(self, dir):
|
||||
"""
|
||||
Fill the DNS resolutions thanks to the dns.log.
|
||||
:return: nothing - all resolutions appended to self.dns.
|
||||
"""
|
||||
if os.path.isfile(os.path.join(dir, "dns.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "dns.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
if record["qtype_name"] in ["A", "AAAA"]:
|
||||
d = {"domain": record["query"],
|
||||
"answers": record["answers"].split(",")}
|
||||
if d not in self.dns:
|
||||
self.dns.append(d)
|
||||
|
||||
def netflow_check(self, dir):
|
||||
"""
|
||||
Enrich and check the netflow from the conn.log against whitelist and IOCs.
|
||||
:return: nothing - all stuff appended to self.alerts
|
||||
"""
|
||||
max_ports = get_config(("analysis", "max_ports"))
|
||||
http_default_port = get_config(("analysis", "http_default_port"))
|
||||
|
||||
# Get the netflow from conn.log.
|
||||
if os.path.isfile(os.path.join(dir, "conn.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "conn.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
c = {"ip_dst": record["id.resp_h"],
|
||||
"proto": record["proto"],
|
||||
"port_dst": record["id.resp_p"],
|
||||
"service": record["service"],
|
||||
"alert_tiggered": False}
|
||||
if c not in self.conns:
|
||||
self.conns.append(c)
|
||||
|
||||
# Let's add some dns resolutions.
|
||||
for c in self.conns:
|
||||
c["resolution"] = self.resolve(c["ip_dst"])
|
||||
|
||||
# Order the conns list by the resolution field.
|
||||
self.conns = sorted(self.conns, key=lambda c: c["resolution"])
|
||||
|
||||
# Check for whitelisted assets, if any, delete the record.
|
||||
if self.whitelist_analysis:
|
||||
|
||||
for i, c in enumerate(self.conns):
|
||||
if c["ip_dst"] in [ip for ip in self.wl_hosts]:
|
||||
self.whitelist.append(self.conns[i])
|
||||
self.conns[i] = False
|
||||
elif c["resolution"] in self.wl_domains:
|
||||
self.whitelist.append(self.conns[i])
|
||||
self.conns[i] = False
|
||||
elif True in [c["resolution"].endswith("." + dom) for dom in self.wl_domains]:
|
||||
self.whitelist.append(self.conns[i])
|
||||
self.conns[i] = False
|
||||
elif True in [IPAddress(c["ip_dst"]) in cidr for cidr in self.wl_cidrs]:
|
||||
self.whitelist.append(self.conns[i])
|
||||
self.conns[i] = False
|
||||
|
||||
# Let's delete whitelisted connections.
|
||||
self.conns = list(filter(lambda c: c != False, self.conns))
|
||||
|
||||
if self.heuristics_analysis:
|
||||
for c in self.conns:
|
||||
# Check for UDP / ICMP (strange from a smartphone.)
|
||||
if c["proto"] in ["UDP", "ICMP"]:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-01"]["title"].format(c["proto"].upper(), c["resolution"]),
|
||||
"description": self.template["PROTO-01"]["description"].format(c["proto"].upper(), c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "PROTO-01"})
|
||||
# Check for use of ports over 1024.
|
||||
if c["port_dst"] >= max_ports:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-02"]["title"].format(c["proto"].upper(), c["resolution"], max_ports),
|
||||
"description": self.template["PROTO-02"]["description"].format(c["proto"].upper(), c["resolution"], c["port_dst"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Low",
|
||||
"id": "PROTO-02"})
|
||||
# Check for use of HTTP.
|
||||
if c["service"] == "http" and c["port_dst"] == http_default_port:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-03"]["title"].format(c["resolution"]),
|
||||
"description": self.template["PROTO-03"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Low",
|
||||
"id": "PROTO-03"})
|
||||
|
||||
# Check for use of HTTP on a non standard port.
|
||||
if c["service"] == "http" and c["port_dst"] != http_default_port:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-04"]["title"].format(c["resolution"], c["port_dst"]),
|
||||
"description": self.template["PROTO-04"]["description"].format(c["resolution"], c["port_dst"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "PROTO-04"})
|
||||
# Check for non-resolved IP address.
|
||||
if c["ip_dst"] == c["resolution"]:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-05"]["title"].format(c["ip_dst"]),
|
||||
"description": self.template["PROTO-05"]["description"].format(c["ip_dst"]),
|
||||
"host": c["ip_dst"],
|
||||
"level": "Low",
|
||||
"id": "PROTO-05"})
|
||||
|
||||
if self.iocs_analysis:
|
||||
|
||||
for c in self.conns:
|
||||
# Check for blacklisted IP address.
|
||||
for host in self.bl_hosts:
|
||||
if c["ip_dst"] == host[0]:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-01"]["title"].format(c["resolution"], c["ip_dst"], host[1].upper()),
|
||||
"description": self.template["IOC-01"]["description"].format(c["ip_dst"]),
|
||||
"host": c["resolution"],
|
||||
"level": "High",
|
||||
"id": "IOC-01"})
|
||||
break
|
||||
# Check for blacklisted CIDR.
|
||||
for cidr in self.bl_cidrs:
|
||||
if IPAddress(c["ip_dst"]) in cidr[0]:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-02"]["title"].format(c["resolution"], cidr[0], cidr[1].upper()),
|
||||
"description": self.template["IOC-02"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "IOC-02"})
|
||||
# Check for blacklisted domain.
|
||||
for domain in self.bl_domains:
|
||||
if c["resolution"].endswith(domain[0]):
|
||||
if domain[1] != "tracker":
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-03"]["title"].format(c["resolution"], domain[1].upper()),
|
||||
"description": self.template["IOC-03"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "High",
|
||||
"id": "IOC-03"})
|
||||
else:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-04"]["title"].format(c["resolution"], domain[1].upper()),
|
||||
"description": self.template["IOC-04"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "IOC-04"})
|
||||
# Check for blacklisted FreeDNS.
|
||||
for domain in self.bl_freedns:
|
||||
if c["resolution"].endswith("." + domain[0]):
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-05"]["title"].format(c["resolution"]),
|
||||
"description": self.template["IOC-05"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "IOC-05"})
|
||||
|
||||
# Check for suspect tlds.
|
||||
for tld in self.bl_tlds:
|
||||
if c["resolution"].endswith(tld[0]):
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-06"]["title"].format(c["resolution"]),
|
||||
"description": self.template["IOC-06"]["description"].format(c["resolution"], tld[0]),
|
||||
"host": c["resolution"],
|
||||
"level": "Low",
|
||||
"id": "IOC-06"})
|
||||
if self.active_analysis:
|
||||
for c in self.conns:
|
||||
try: # Domain nameservers check.
|
||||
name_servers = pydig.query(c["resolution"], "NS")
|
||||
if len(name_servers):
|
||||
for ns in self.bl_nameservers:
|
||||
if name_servers[0].endswith(".{}.".format(ns[0])):
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["ACT-01"]["title"].format(c["resolution"], name_servers[0]),
|
||||
"description": self.template["ACT-01"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "ACT-01"})
|
||||
except:
|
||||
pass
|
||||
|
||||
try: # Domain history check.
|
||||
|
||||
whois_record = whois.whois(c["resolution"])
|
||||
creation_date = whois_record.creation_date if type(
|
||||
whois_record.creation_date) is not list else whois_record.creation_date[0]
|
||||
creation_days = abs((datetime.now() - creation_date).days)
|
||||
if creation_days < 365:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["ACT-02"]["title"].format(c["resolution"], creation_days),
|
||||
"description": self.template["ACT-02"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "ACT-02"})
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
def files_check(self, dir):
|
||||
"""
|
||||
Check on the files.log:
|
||||
* Check certificates sha1
|
||||
* [todo] Check possible binary data or APKs?
|
||||
:return: nothing - all stuff appended to self.alerts
|
||||
"""
|
||||
|
||||
if not self.iocs_analysis:
|
||||
return
|
||||
|
||||
bl_certs = get_iocs("sha1cert")
|
||||
|
||||
if os.path.isfile(os.path.join(dir, "files.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "files.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
f = {"filename": record["filename"],
|
||||
"ip_src": record["id.orig_h"],
|
||||
"ip_dst": record["id.resp_h"],
|
||||
"mime_type": record["mime_type"],
|
||||
"sha1": record["sha1"]}
|
||||
if f not in self.files:
|
||||
self.files.append(f)
|
||||
|
||||
for f in self.files:
|
||||
if f["mime_type"] == "application/x-x509-user-cert":
|
||||
for cert in bl_certs: # Check for blacklisted certificate.
|
||||
if f["sha1"] == cert[0]:
|
||||
host = self.resolve(f["ip_src"])
|
||||
self.alerts.append({"title": self.template["IOC-07"]["title"].format(cert[1].upper(), host),
|
||||
"description": self.template["IOC-07"]["description"].format(f["sha1"], host),
|
||||
"host": host,
|
||||
"level": "High",
|
||||
"id": "IOC-07"})
|
||||
|
||||
def http_check(self, dir):
|
||||
"""
|
||||
Check on the http.log:
|
||||
* Blacklisted domain/tld from the Host header field.
|
||||
Can be used when no DNS query have been done during the session (already cached by the device.)
|
||||
:return: nothing - all stuff appended to self.alerts
|
||||
"""
|
||||
|
||||
if os.path.isfile(os.path.join(dir, "http.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "http.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
c = {"host": record['host']}
|
||||
if c not in self.http:
|
||||
self.http.append(c)
|
||||
|
||||
if self.iocs_analysis:
|
||||
for c in self.http:
|
||||
|
||||
# If we already know the host form DNS, let's continue.
|
||||
if c["host"] in [r["domain"] for r in self.dns]:
|
||||
continue
|
||||
|
||||
# Check for blacklisted domain.
|
||||
for h in self.bl_domains:
|
||||
if h[1] != "tracker":
|
||||
if c["host"].endswith(h[0]):
|
||||
self.alerts.append({"title": self.template["IOC-08"]["title"].format(c["host"], h[1].upper()),
|
||||
"description": self.template["IOC-08"]["description"].format(c["host"]),
|
||||
"host": c["host"],
|
||||
"level": "High",
|
||||
"id": "IOC-08"})
|
||||
# Check for freedns.
|
||||
for h in self.bl_freedns:
|
||||
if c["host"].endswith(h[0]):
|
||||
self.alerts.append({"title": self.template["IOC-09"]["title"].format(c["host"]),
|
||||
"description": self.template["IOC-09"]["description"].format(c["host"]),
|
||||
"host": c["host"],
|
||||
"level": "Moderate",
|
||||
"id": "IOC-09"})
|
||||
# Check for fancy TLD.
|
||||
for h in self.bl_tlds:
|
||||
if c["host"].endswith(h[0]):
|
||||
self.alerts.append({"title": self.template["IOC-10"]["title"].format(c["host"]),
|
||||
"description": self.template["IOC-10"]["description"].format(c["host"], h[0]),
|
||||
"host": c["host"],
|
||||
"level": "Low",
|
||||
"id": "IOC-10"})
|
||||
|
||||
def ssl_check(self, dir):
|
||||
"""
|
||||
Check on the ssl.log:
|
||||
* SSL connections which doesn't use the 443.
|
||||
* "Free" certificate issuer (taken from the config).
|
||||
* Self-signed certificates.
|
||||
* Blacklisted domain in the CN
|
||||
:return: nothing - all stuff appended to self.alerts
|
||||
"""
|
||||
ssl_default_ports = get_config(("analysis", "ssl_default_ports"))
|
||||
free_issuers = get_config(("analysis", "free_issuers"))
|
||||
|
||||
if os.path.isfile(os.path.join(dir, "ssl.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "ssl.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
c = {"host": record['id.resp_h'],
|
||||
"port": record['id.resp_p'],
|
||||
"issuer": record["issuer"] if "issuer" in record else "",
|
||||
"validation_status": record["validation_status"],
|
||||
"cn": record["server_name"] if "server_name" in record else ""}
|
||||
if c not in self.ssl:
|
||||
self.ssl.append(c)
|
||||
|
||||
if self.heuristics_analysis:
|
||||
for cert in self.ssl:
|
||||
host = self.resolve(cert["host"])
|
||||
|
||||
# If the associated host has not whitelisted, check the cert.
|
||||
for c in self.conns:
|
||||
if host in c["resolution"]:
|
||||
# Check for non generic SSL port.
|
||||
if cert["port"] not in ssl_default_ports:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["SSL-01"]["title"].format(cert["port"], host),
|
||||
"description": self.template["SSL-01"]["description"].format(host),
|
||||
"host": host,
|
||||
"level": "Moderate",
|
||||
"id": "SSL-01"})
|
||||
# Check Free SSL certificates.
|
||||
if cert["issuer"] in free_issuers:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["SSL-02"]["title"].format(host),
|
||||
"description": self.template["SSL-02"]["description"],
|
||||
"host": host,
|
||||
"level": "Moderate",
|
||||
"id": "SSL-02"})
|
||||
# Check for self-signed certificates.
|
||||
if cert["validation_status"] == "self signed certificate in certificate chain":
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["SSL-03"]["title"].format(host),
|
||||
"description": self.template["SSL-03"]["description"].format(host),
|
||||
"host": host,
|
||||
"level": "Moderate",
|
||||
"id": "SSL-03"})
|
||||
|
||||
if self.iocs_analysis:
|
||||
for cert in self.ssl:
|
||||
# Check if the domain in the certificate haven't been blacklisted
|
||||
# This check can be good if the domain has already been cached by
|
||||
# the device so it wont appear in self.dns.
|
||||
|
||||
if any([cert["cn"].endswith(r["domain"]) for r in self.dns]):
|
||||
continue
|
||||
|
||||
for domain in self.bl_domains:
|
||||
if domain[1] != "tracker":
|
||||
if cert["cn"].endswith(domain[0]):
|
||||
self.alerts.append({"title": self.template["SSL-04"]["title"].format(domain[0], domain[1].upper()),
|
||||
"description": self.template["SSL-04"]["description"].format(domain[0]),
|
||||
"host": domain[0],
|
||||
"level": "High",
|
||||
"id": "SSL-04"})
|
||||
|
||||
def alerts_check(self):
|
||||
"""
|
||||
Leverage an advice to the user based on the trigered hosts
|
||||
:return: nothing - all generated alerts appended to self.alerts
|
||||
"""
|
||||
hosts = {}
|
||||
|
||||
for alert in [dict(t) for t in {tuple(d.items()) for d in self.alerts}]:
|
||||
if alert["host"] not in hosts:
|
||||
hosts[alert["host"]] = 1
|
||||
else:
|
||||
hosts[alert["host"]] += 1
|
||||
|
||||
for host, nb in hosts.items():
|
||||
if nb >= get_config(("analysis", "max_alerts")):
|
||||
self.alerts.append({"title": self.template["ADV-01"]["title"].format(host),
|
||||
"description": self.template["ADV-01"]["description"].format(host, nb),
|
||||
"host": host,
|
||||
"level": "Moderate",
|
||||
"id": "ADV-01"})
|
||||
|
||||
def resolve(self, ip_addr):
|
||||
"""
|
||||
A simple method to retreive DNS names from IP addresses
|
||||
in order to replace them in alerts.
|
||||
|
||||
:return: String - DNS record or IP Address.
|
||||
"""
|
||||
for record in self.dns:
|
||||
if ip_addr in record["answers"]:
|
||||
return record["domain"]
|
||||
return ip_addr
|
||||
|
||||
def start_zeek(self):
|
||||
"""
|
||||
Start zeek and check the logs.
|
||||
"""
|
||||
sp.Popen("cd {} && /opt/zeek/bin/zeek -Cr capture.pcap protocols/ssl/validate-certs".format(
|
||||
self.working_dir), shell=True).wait()
|
||||
sp.Popen("cd {} && mv *.log assets/".format(self.working_dir),
|
||||
shell=True).wait()
|
||||
self.fill_dns(self.working_dir + "/assets/")
|
||||
self.netflow_check(self.working_dir + "/assets/")
|
||||
self.ssl_check(self.working_dir + "/assets/")
|
||||
self.http_check(self.working_dir + "/assets/")
|
||||
self.files_check(self.working_dir + "/assets/")
|
||||
self.alerts_check()
|
||||
|
||||
def retrieve_alerts(self):
|
||||
"""
|
||||
Retrieve alerts.
|
||||
:return: list - a list of alerts wihout duplicates.
|
||||
"""
|
||||
return [dict(t) for t in {tuple(d.items()) for d in self.alerts}]
|
||||
|
||||
def retrieve_whitelist(self):
|
||||
"""
|
||||
Retrieve whitelisted elements.
|
||||
:return: list - a list of whitelisted elements wihout duplicates.
|
||||
"""
|
||||
return [dict(t) for t in {tuple(d.items()) for d in self.whitelist}]
|
||||
|
||||
def retrieve_conns(self):
|
||||
"""
|
||||
Retrieve not whitelisted elements.
|
||||
:return: list - a list of non-whitelisted elements wihout duplicates.
|
||||
"""
|
||||
return self.conns
|
||||
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from classes.parsezeeklogs import ParseZeekLogs
|
||||
from netaddr import IPNetwork, IPAddress
|
||||
from utils import get_iocs, get_config, get_whitelist
|
||||
from datetime import datetime
|
||||
import subprocess as sp
|
||||
import json
|
||||
import pydig
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import whois
|
||||
from security import safe_command
|
||||
|
||||
|
||||
class ZeekEngine(object):
|
||||
|
||||
def __init__(self, capture_directory):
|
||||
self.working_dir = capture_directory
|
||||
self.alerts = []
|
||||
self.conns = []
|
||||
self.ssl = []
|
||||
self.http = []
|
||||
self.dns = []
|
||||
self.files = []
|
||||
self.whitelist = []
|
||||
|
||||
# Get analysis and userlang configuration
|
||||
self.heuristics_analysis = get_config(("analysis", "heuristics"))
|
||||
self.iocs_analysis = get_config(("analysis", "iocs"))
|
||||
self.whitelist_analysis = get_config(("analysis", "whitelist"))
|
||||
self.active_analysis = get_config(("analysis", "active"))
|
||||
self.userlang = get_config(("frontend", "user_lang"))
|
||||
|
||||
# Retreive IOCs.
|
||||
if self.iocs_analysis:
|
||||
self.bl_cidrs = [[IPNetwork(cidr[0]), cidr[1]]
|
||||
for cidr in get_iocs("cidr")]
|
||||
self.bl_hosts = get_iocs("ip4addr") + get_iocs("ip6addr")
|
||||
self.bl_domains = get_iocs("domain")
|
||||
self.bl_freedns = get_iocs("freedns")
|
||||
self.bl_nameservers = get_iocs("ns")
|
||||
self.bl_tlds = get_iocs("tld")
|
||||
|
||||
# Retreive whitelisted items.
|
||||
if self.whitelist_analysis:
|
||||
self.wl_cidrs = [IPNetwork(cidr) for cidr in get_whitelist("cidr")]
|
||||
self.wl_hosts = get_whitelist("ip4addr") + get_whitelist("ip6addr")
|
||||
self.wl_domains = get_whitelist("domain")
|
||||
|
||||
# Load template language
|
||||
if not re.match("^[a-z]{2,3}$", self.userlang):
|
||||
self.userlang = "en"
|
||||
with open(os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "locales/{}.json".format(self.userlang))) as f:
|
||||
self.template = json.load(f)["alerts"]
|
||||
|
||||
def fill_dns(self, dir):
|
||||
"""
|
||||
Fill the DNS resolutions thanks to the dns.log.
|
||||
:return: nothing - all resolutions appended to self.dns.
|
||||
"""
|
||||
if os.path.isfile(os.path.join(dir, "dns.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "dns.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
if record["qtype_name"] in ["A", "AAAA"]:
|
||||
d = {"domain": record["query"],
|
||||
"answers": record["answers"].split(",")}
|
||||
if d not in self.dns:
|
||||
self.dns.append(d)
|
||||
|
||||
def netflow_check(self, dir):
|
||||
"""
|
||||
Enrich and check the netflow from the conn.log against whitelist and IOCs.
|
||||
:return: nothing - all stuff appended to self.alerts
|
||||
"""
|
||||
max_ports = get_config(("analysis", "max_ports"))
|
||||
http_default_port = get_config(("analysis", "http_default_port"))
|
||||
|
||||
# Get the netflow from conn.log.
|
||||
if os.path.isfile(os.path.join(dir, "conn.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "conn.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
c = {"ip_dst": record["id.resp_h"],
|
||||
"proto": record["proto"],
|
||||
"port_dst": record["id.resp_p"],
|
||||
"service": record["service"],
|
||||
"alert_tiggered": False}
|
||||
if c not in self.conns:
|
||||
self.conns.append(c)
|
||||
|
||||
# Let's add some dns resolutions.
|
||||
for c in self.conns:
|
||||
c["resolution"] = self.resolve(c["ip_dst"])
|
||||
|
||||
# Order the conns list by the resolution field.
|
||||
self.conns = sorted(self.conns, key=lambda c: c["resolution"])
|
||||
|
||||
# Check for whitelisted assets, if any, delete the record.
|
||||
if self.whitelist_analysis:
|
||||
|
||||
for i, c in enumerate(self.conns):
|
||||
if c["ip_dst"] in [ip for ip in self.wl_hosts]:
|
||||
self.whitelist.append(self.conns[i])
|
||||
self.conns[i] = False
|
||||
elif c["resolution"] in self.wl_domains:
|
||||
self.whitelist.append(self.conns[i])
|
||||
self.conns[i] = False
|
||||
elif True in [c["resolution"].endswith("." + dom) for dom in self.wl_domains]:
|
||||
self.whitelist.append(self.conns[i])
|
||||
self.conns[i] = False
|
||||
elif True in [IPAddress(c["ip_dst"]) in cidr for cidr in self.wl_cidrs]:
|
||||
self.whitelist.append(self.conns[i])
|
||||
self.conns[i] = False
|
||||
|
||||
# Let's delete whitelisted connections.
|
||||
self.conns = list(filter(lambda c: c != False, self.conns))
|
||||
|
||||
if self.heuristics_analysis:
|
||||
for c in self.conns:
|
||||
# Check for UDP / ICMP (strange from a smartphone.)
|
||||
if c["proto"] in ["UDP", "ICMP"]:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-01"]["title"].format(c["proto"].upper(), c["resolution"]),
|
||||
"description": self.template["PROTO-01"]["description"].format(c["proto"].upper(), c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "PROTO-01"})
|
||||
# Check for use of ports over 1024.
|
||||
if c["port_dst"] >= max_ports:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-02"]["title"].format(c["proto"].upper(), c["resolution"], max_ports),
|
||||
"description": self.template["PROTO-02"]["description"].format(c["proto"].upper(), c["resolution"], c["port_dst"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Low",
|
||||
"id": "PROTO-02"})
|
||||
# Check for use of HTTP.
|
||||
if c["service"] == "http" and c["port_dst"] == http_default_port:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-03"]["title"].format(c["resolution"]),
|
||||
"description": self.template["PROTO-03"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Low",
|
||||
"id": "PROTO-03"})
|
||||
|
||||
# Check for use of HTTP on a non standard port.
|
||||
if c["service"] == "http" and c["port_dst"] != http_default_port:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-04"]["title"].format(c["resolution"], c["port_dst"]),
|
||||
"description": self.template["PROTO-04"]["description"].format(c["resolution"], c["port_dst"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "PROTO-04"})
|
||||
# Check for non-resolved IP address.
|
||||
if c["ip_dst"] == c["resolution"]:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["PROTO-05"]["title"].format(c["ip_dst"]),
|
||||
"description": self.template["PROTO-05"]["description"].format(c["ip_dst"]),
|
||||
"host": c["ip_dst"],
|
||||
"level": "Low",
|
||||
"id": "PROTO-05"})
|
||||
|
||||
if self.iocs_analysis:
|
||||
|
||||
for c in self.conns:
|
||||
# Check for blacklisted IP address.
|
||||
for host in self.bl_hosts:
|
||||
if c["ip_dst"] == host[0]:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-01"]["title"].format(c["resolution"], c["ip_dst"], host[1].upper()),
|
||||
"description": self.template["IOC-01"]["description"].format(c["ip_dst"]),
|
||||
"host": c["resolution"],
|
||||
"level": "High",
|
||||
"id": "IOC-01"})
|
||||
break
|
||||
# Check for blacklisted CIDR.
|
||||
for cidr in self.bl_cidrs:
|
||||
if IPAddress(c["ip_dst"]) in cidr[0]:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-02"]["title"].format(c["resolution"], cidr[0], cidr[1].upper()),
|
||||
"description": self.template["IOC-02"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "IOC-02"})
|
||||
# Check for blacklisted domain.
|
||||
for domain in self.bl_domains:
|
||||
if c["resolution"].endswith(domain[0]):
|
||||
if domain[1] != "tracker":
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-03"]["title"].format(c["resolution"], domain[1].upper()),
|
||||
"description": self.template["IOC-03"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "High",
|
||||
"id": "IOC-03"})
|
||||
else:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-04"]["title"].format(c["resolution"], domain[1].upper()),
|
||||
"description": self.template["IOC-04"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "IOC-04"})
|
||||
# Check for blacklisted FreeDNS.
|
||||
for domain in self.bl_freedns:
|
||||
if c["resolution"].endswith("." + domain[0]):
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-05"]["title"].format(c["resolution"]),
|
||||
"description": self.template["IOC-05"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "IOC-05"})
|
||||
|
||||
# Check for suspect tlds.
|
||||
for tld in self.bl_tlds:
|
||||
if c["resolution"].endswith(tld[0]):
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["IOC-06"]["title"].format(c["resolution"]),
|
||||
"description": self.template["IOC-06"]["description"].format(c["resolution"], tld[0]),
|
||||
"host": c["resolution"],
|
||||
"level": "Low",
|
||||
"id": "IOC-06"})
|
||||
if self.active_analysis:
|
||||
for c in self.conns:
|
||||
try: # Domain nameservers check.
|
||||
name_servers = pydig.query(c["resolution"], "NS")
|
||||
if len(name_servers):
|
||||
for ns in self.bl_nameservers:
|
||||
if name_servers[0].endswith(".{}.".format(ns[0])):
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["ACT-01"]["title"].format(c["resolution"], name_servers[0]),
|
||||
"description": self.template["ACT-01"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "ACT-01"})
|
||||
except:
|
||||
pass
|
||||
|
||||
try: # Domain history check.
|
||||
|
||||
whois_record = whois.whois(c["resolution"])
|
||||
creation_date = whois_record.creation_date if type(
|
||||
whois_record.creation_date) is not list else whois_record.creation_date[0]
|
||||
creation_days = abs((datetime.now() - creation_date).days)
|
||||
if creation_days < 365:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["ACT-02"]["title"].format(c["resolution"], creation_days),
|
||||
"description": self.template["ACT-02"]["description"].format(c["resolution"]),
|
||||
"host": c["resolution"],
|
||||
"level": "Moderate",
|
||||
"id": "ACT-02"})
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
def files_check(self, dir):
|
||||
"""
|
||||
Check on the files.log:
|
||||
* Check certificates sha1
|
||||
* [todo] Check possible binary data or APKs?
|
||||
:return: nothing - all stuff appended to self.alerts
|
||||
"""
|
||||
|
||||
if not self.iocs_analysis:
|
||||
return
|
||||
|
||||
bl_certs = get_iocs("sha1cert")
|
||||
|
||||
if os.path.isfile(os.path.join(dir, "files.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "files.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
f = {"filename": record["filename"],
|
||||
"ip_src": record["id.orig_h"],
|
||||
"ip_dst": record["id.resp_h"],
|
||||
"mime_type": record["mime_type"],
|
||||
"sha1": record["sha1"]}
|
||||
if f not in self.files:
|
||||
self.files.append(f)
|
||||
|
||||
for f in self.files:
|
||||
if f["mime_type"] == "application/x-x509-user-cert":
|
||||
for cert in bl_certs: # Check for blacklisted certificate.
|
||||
if f["sha1"] == cert[0]:
|
||||
host = self.resolve(f["ip_src"])
|
||||
self.alerts.append({"title": self.template["IOC-07"]["title"].format(cert[1].upper(), host),
|
||||
"description": self.template["IOC-07"]["description"].format(f["sha1"], host),
|
||||
"host": host,
|
||||
"level": "High",
|
||||
"id": "IOC-07"})
|
||||
|
||||
def http_check(self, dir):
|
||||
"""
|
||||
Check on the http.log:
|
||||
* Blacklisted domain/tld from the Host header field.
|
||||
Can be used when no DNS query have been done during the session (already cached by the device.)
|
||||
:return: nothing - all stuff appended to self.alerts
|
||||
"""
|
||||
|
||||
if os.path.isfile(os.path.join(dir, "http.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "http.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
c = {"host": record['host']}
|
||||
if c not in self.http:
|
||||
self.http.append(c)
|
||||
|
||||
if self.iocs_analysis:
|
||||
for c in self.http:
|
||||
|
||||
# If we already know the host form DNS, let's continue.
|
||||
if c["host"] in [r["domain"] for r in self.dns]:
|
||||
continue
|
||||
|
||||
# Check for blacklisted domain.
|
||||
for h in self.bl_domains:
|
||||
if h[1] != "tracker":
|
||||
if c["host"].endswith(h[0]):
|
||||
self.alerts.append({"title": self.template["IOC-08"]["title"].format(c["host"], h[1].upper()),
|
||||
"description": self.template["IOC-08"]["description"].format(c["host"]),
|
||||
"host": c["host"],
|
||||
"level": "High",
|
||||
"id": "IOC-08"})
|
||||
# Check for freedns.
|
||||
for h in self.bl_freedns:
|
||||
if c["host"].endswith(h[0]):
|
||||
self.alerts.append({"title": self.template["IOC-09"]["title"].format(c["host"]),
|
||||
"description": self.template["IOC-09"]["description"].format(c["host"]),
|
||||
"host": c["host"],
|
||||
"level": "Moderate",
|
||||
"id": "IOC-09"})
|
||||
# Check for fancy TLD.
|
||||
for h in self.bl_tlds:
|
||||
if c["host"].endswith(h[0]):
|
||||
self.alerts.append({"title": self.template["IOC-10"]["title"].format(c["host"]),
|
||||
"description": self.template["IOC-10"]["description"].format(c["host"], h[0]),
|
||||
"host": c["host"],
|
||||
"level": "Low",
|
||||
"id": "IOC-10"})
|
||||
|
||||
def ssl_check(self, dir):
|
||||
"""
|
||||
Check on the ssl.log:
|
||||
* SSL connections which doesn't use the 443.
|
||||
* "Free" certificate issuer (taken from the config).
|
||||
* Self-signed certificates.
|
||||
* Blacklisted domain in the CN
|
||||
:return: nothing - all stuff appended to self.alerts
|
||||
"""
|
||||
ssl_default_ports = get_config(("analysis", "ssl_default_ports"))
|
||||
free_issuers = get_config(("analysis", "free_issuers"))
|
||||
|
||||
if os.path.isfile(os.path.join(dir, "ssl.log")):
|
||||
for record in ParseZeekLogs(os.path.join(dir, "ssl.log"), output_format="json", safe_headers=False):
|
||||
if record is not None:
|
||||
c = {"host": record['id.resp_h'],
|
||||
"port": record['id.resp_p'],
|
||||
"issuer": record["issuer"] if "issuer" in record else "",
|
||||
"validation_status": record["validation_status"],
|
||||
"cn": record["server_name"] if "server_name" in record else ""}
|
||||
if c not in self.ssl:
|
||||
self.ssl.append(c)
|
||||
|
||||
if self.heuristics_analysis:
|
||||
for cert in self.ssl:
|
||||
host = self.resolve(cert["host"])
|
||||
|
||||
# If the associated host has not whitelisted, check the cert.
|
||||
for c in self.conns:
|
||||
if host in c["resolution"]:
|
||||
# Check for non generic SSL port.
|
||||
if cert["port"] not in ssl_default_ports:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["SSL-01"]["title"].format(cert["port"], host),
|
||||
"description": self.template["SSL-01"]["description"].format(host),
|
||||
"host": host,
|
||||
"level": "Moderate",
|
||||
"id": "SSL-01"})
|
||||
# Check Free SSL certificates.
|
||||
if cert["issuer"] in free_issuers:
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["SSL-02"]["title"].format(host),
|
||||
"description": self.template["SSL-02"]["description"],
|
||||
"host": host,
|
||||
"level": "Moderate",
|
||||
"id": "SSL-02"})
|
||||
# Check for self-signed certificates.
|
||||
if cert["validation_status"] == "self signed certificate in certificate chain":
|
||||
c["alert_tiggered"] = True
|
||||
self.alerts.append({"title": self.template["SSL-03"]["title"].format(host),
|
||||
"description": self.template["SSL-03"]["description"].format(host),
|
||||
"host": host,
|
||||
"level": "Moderate",
|
||||
"id": "SSL-03"})
|
||||
|
||||
if self.iocs_analysis:
|
||||
for cert in self.ssl:
|
||||
# Check if the domain in the certificate haven't been blacklisted
|
||||
# This check can be good if the domain has already been cached by
|
||||
# the device so it wont appear in self.dns.
|
||||
|
||||
if any(cert["cn"].endswith(r["domain"]) for r in self.dns):
|
||||
continue
|
||||
|
||||
for domain in self.bl_domains:
|
||||
if domain[1] != "tracker":
|
||||
if cert["cn"].endswith(domain[0]):
|
||||
self.alerts.append({"title": self.template["SSL-04"]["title"].format(domain[0], domain[1].upper()),
|
||||
"description": self.template["SSL-04"]["description"].format(domain[0]),
|
||||
"host": domain[0],
|
||||
"level": "High",
|
||||
"id": "SSL-04"})
|
||||
|
||||
def alerts_check(self):
|
||||
"""
|
||||
Leverage an advice to the user based on the trigered hosts
|
||||
:return: nothing - all generated alerts appended to self.alerts
|
||||
"""
|
||||
hosts = {}
|
||||
|
||||
for alert in [dict(t) for t in {tuple(d.items()) for d in self.alerts}]:
|
||||
if alert["host"] not in hosts:
|
||||
hosts[alert["host"]] = 1
|
||||
else:
|
||||
hosts[alert["host"]] += 1
|
||||
|
||||
for host, nb in hosts.items():
|
||||
if nb >= get_config(("analysis", "max_alerts")):
|
||||
self.alerts.append({"title": self.template["ADV-01"]["title"].format(host),
|
||||
"description": self.template["ADV-01"]["description"].format(host, nb),
|
||||
"host": host,
|
||||
"level": "Moderate",
|
||||
"id": "ADV-01"})
|
||||
|
||||
def resolve(self, ip_addr):
|
||||
"""
|
||||
A simple method to retreive DNS names from IP addresses
|
||||
in order to replace them in alerts.
|
||||
|
||||
:return: String - DNS record or IP Address.
|
||||
"""
|
||||
for record in self.dns:
|
||||
if ip_addr in record["answers"]:
|
||||
return record["domain"]
|
||||
return ip_addr
|
||||
|
||||
def start_zeek(self):
|
||||
"""
|
||||
Start zeek and check the logs.
|
||||
"""
|
||||
safe_command.run(sp.Popen, "cd {} && /opt/zeek/bin/zeek -Cr capture.pcap protocols/ssl/validate-certs".format(
|
||||
self.working_dir), shell=False).wait()
|
||||
safe_command.run(sp.Popen, "cd {} && mv *.log assets/".format(self.working_dir),
|
||||
shell=False).wait()
|
||||
self.fill_dns(self.working_dir + "/assets/")
|
||||
self.netflow_check(self.working_dir + "/assets/")
|
||||
self.ssl_check(self.working_dir + "/assets/")
|
||||
self.http_check(self.working_dir + "/assets/")
|
||||
self.files_check(self.working_dir + "/assets/")
|
||||
self.alerts_check()
|
||||
|
||||
def retrieve_alerts(self):
|
||||
"""
|
||||
Retrieve alerts.
|
||||
:return: list - a list of alerts wihout duplicates.
|
||||
"""
|
||||
return [dict(t) for t in {tuple(d.items()) for d in self.alerts}]
|
||||
|
||||
def retrieve_whitelist(self):
|
||||
"""
|
||||
Retrieve whitelisted elements.
|
||||
:return: list - a list of whitelisted elements wihout duplicates.
|
||||
"""
|
||||
return [dict(t) for t in {tuple(d.items()) for d in self.whitelist}]
|
||||
|
||||
def retrieve_conns(self):
|
||||
"""
|
||||
Retrieve not whitelisted elements.
|
||||
:return: list - a list of non-whitelisted elements wihout duplicates.
|
||||
"""
|
||||
return self.conns
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 674 KiB After Width: | Height: | Size: 805 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 660 KiB After Width: | Height: | Size: 784 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 3.5 KiB After Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 185 KiB After Width: | Height: | Size: 315 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 92 KiB After Width: | Height: | Size: 133 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 674 KiB After Width: | Height: | Size: 805 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 660 KiB After Width: | Height: | Size: 784 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 28 KiB |
@@ -1 +1,4 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="112" height="195" fill="none" viewBox="0 0 112 195"><line x1="3.5" x2="3.5" y1="3.5" y2="191.5" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="7"/><rect width="105" height="195" x="7" fill="#F7F8F9"/></svg>
|
||||
<svg width="112" height="195" viewBox="0 0 112 195" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<line x1="3.5" y1="3.5" x2="3.50001" y2="191.5" stroke="black" stroke-width="7" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<rect x="7" width="105" height="195" fill="#F7F8F9"/>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 281 B After Width: | Height: | Size: 294 B |
@@ -1 +1,16 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="loader-1" width="40" height="40" x="0" y="0" enable-background="new 0 0 40 40" version="1.1" viewBox="0 0 40 40" xml:space="preserve"><path fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946 s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634 c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z" opacity=".2"/><path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0 C22.32,8.481,24.301,9.057,26.013,10.047z"><animateTransform attributeName="transform" attributeType="xml" dur="0.5s" from="0 20 20" repeatCount="indefinite" to="360 20 20" type="rotate"/></path></svg>
|
||||
<svg version="1.1" id="loader-1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
width="40px" height="40px" viewBox="0 0 40 40" enable-background="new 0 0 40 40" xml:space="preserve">
|
||||
<path opacity="0.2" fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946
|
||||
s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634
|
||||
c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z"/>
|
||||
<path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0
|
||||
C22.32,8.481,24.301,9.057,26.013,10.047z">
|
||||
<animateTransform attributeType="xml"
|
||||
attributeName="transform"
|
||||
type="rotate"
|
||||
from="0 20 20"
|
||||
to="360 20 20"
|
||||
dur="0.5s"
|
||||
repeatCount="indefinite"/>
|
||||
</path>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 874 B After Width: | Height: | Size: 970 B |
@@ -1 +1,5 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="106" fill="none" viewBox="0 0 106 106"><circle cx="53" cy="53" r="53" fill="#40D8A1"/><path stroke="#fff" stroke-linecap="round" stroke-linejoin="round" stroke-width="10" d="M29 52.5L47.5 70.5"/><line x1="79" x2="48.071" y1="40.071" y2="71" stroke="#fff" stroke-linecap="round" stroke-width="10"/></svg>
|
||||
<svg width="106" height="106" viewBox="0 0 106 106" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<circle cx="53" cy="53" r="53" fill="#40D8A1"/>
|
||||
<path d="M29 52.5L47.5 70.5" stroke="white" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<line x1="79" y1="40.0711" x2="48.0711" y2="71" stroke="white" stroke-width="10" stroke-linecap="round"/>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 363 B After Width: | Height: | Size: 377 B |
@@ -1 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="548" height="199" fill="none" viewBox="0 0 548 199"><rect width="142" height="145" x="403" y="27" fill="#fff" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="6" rx="8"/><path fill="#000" d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z"/><rect width="26" height="26" x="477" y="55" fill="#fff" stroke="#000" stroke-width="6"/><rect width="26" height="26" x="477" y="117" fill="#fff" stroke="#000" stroke-width="6"/></svg>
|
||||
<svg width="548" height="199" viewBox="0 0 548 199" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect x="403" y="27" width="142" height="145" rx="8" fill="white" stroke="black" stroke-width="6" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z" fill="black"/>
|
||||
<rect x="477" y="55" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
|
||||
<rect x="477" y="117" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 583 B After Width: | Height: | Size: 602 B |
@@ -1 +1,11 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin:auto;background:0 0;display:block;shape-rendering:auto" width="200" height="200" preserveAspectRatio="xMidYMid" viewBox="0 0 100 100"><circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2"><animate attributeName="r" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle><circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2"><animate attributeName="r" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle></svg>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin: auto; background: none; display: block; shape-rendering: auto;" width="200px" height="200px" viewBox="0 0 100 100" preserveAspectRatio="xMidYMid">
|
||||
<circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2">
|
||||
<animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
|
||||
<animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
|
||||
</circle>
|
||||
<circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2">
|
||||
<animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline"></animate>
|
||||
<animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline"></animate>
|
||||
</circle>
|
||||
<!-- [ldio] generated by https://loading.io/ --></svg>
|
||||
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB After Width: | Height: | Size: 88 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 540 KiB After Width: | Height: | Size: 775 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 324 KiB After Width: | Height: | Size: 405 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 185 KiB After Width: | Height: | Size: 315 KiB |
+20
-17
@@ -1,17 +1,20 @@
|
||||
pymisp==2.4.165.1
|
||||
sqlalchemy==1.4.48
|
||||
ipwhois==1.2.0
|
||||
netaddr==0.8.0
|
||||
flask==2.2.5
|
||||
flask_httpauth==4.8.0
|
||||
pyjwt==2.4.0
|
||||
psutil==5.8.0
|
||||
pydig==0.4.0
|
||||
pyudev==0.24.0
|
||||
pyyaml==5.3.1
|
||||
wifi==0.3.8
|
||||
qrcode==7.3.1
|
||||
netifaces==0.11.0
|
||||
weasyprint==59.0
|
||||
python-whois==0.8.0
|
||||
six==1.16.0
|
||||
pymisp==2.4.165.1
|
||||
sqlalchemy==1.4.48
|
||||
ipwhois==1.2.0
|
||||
netaddr==0.8.0
|
||||
flask==1.1.2
|
||||
flask_httpauth==4.8.0
|
||||
pyjwt==1.7.1
|
||||
psutil==5.8.0
|
||||
pydig==0.4.0
|
||||
pyudev==0.24.0
|
||||
pyyaml==5.3.1
|
||||
wifi==0.3.8
|
||||
qrcode==7.3.1
|
||||
netifaces==0.11.0
|
||||
weasyprint==59.0
|
||||
python-whois==0.8.0
|
||||
six==1.16.0
|
||||
security==1.2.1 \
|
||||
--hash=sha256:4ca5f8cfc6b836e2192a84bb5a28b72c17f3cd1abbfe3281f917394c6e6c9238
|
||||
--hash=sha256:0a9dc7b457330e6d0f92bdae3603fecb85394beefad0fd3b5058758a58781ded
|
||||
|
||||
+149
-149
@@ -1,149 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from app.utils import read_config
|
||||
from app.classes.iocs import IOCs
|
||||
from app.classes.whitelist import WhiteList
|
||||
from app.classes.misp import MISP
|
||||
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
import time
|
||||
from multiprocessing import Process
|
||||
|
||||
"""
|
||||
This file is parsing the watchers present
|
||||
in the configuration file. This in order to get
|
||||
automatically new iocs / elements from remote
|
||||
sources without user interaction.
|
||||
"""
|
||||
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
|
||||
def watch_iocs():
|
||||
"""
|
||||
Retrieve IOCs from the remote URLs defined in config/watchers.
|
||||
For each IOC, add it to the DB.
|
||||
"""
|
||||
|
||||
# Retrieve the URLs from the configuration
|
||||
urls = read_config(("watchers", "iocs"))
|
||||
watchers = [{"url": url, "status": False} for url in urls]
|
||||
|
||||
while True:
|
||||
for w in watchers:
|
||||
if w["status"] == False:
|
||||
iocs = IOCs()
|
||||
iocs_list = []
|
||||
to_delete = []
|
||||
try:
|
||||
res = requests.get(w["url"], verify=True)
|
||||
if res.status_code == 200:
|
||||
content = json.loads(res.content)
|
||||
iocs_list = content["iocs"] if "iocs" in content else []
|
||||
to_delete = content["to_delete"] if "to_delete" in content else []
|
||||
else:
|
||||
w["status"] = False
|
||||
except:
|
||||
w["status"] = False
|
||||
|
||||
for ioc in iocs_list:
|
||||
try:
|
||||
iocs.add(ioc["type"], ioc["tag"],
|
||||
ioc["tlp"], ioc["value"], "watcher")
|
||||
w["status"] = True
|
||||
except:
|
||||
continue
|
||||
|
||||
for ioc in to_delete:
|
||||
try:
|
||||
iocs.delete_by_value(ioc["value"])
|
||||
w["status"] = True
|
||||
except:
|
||||
continue
|
||||
|
||||
# If at least one URL haven't be parsed, let's retry in 1min.
|
||||
if False in [w["status"] for w in watchers]:
|
||||
time.sleep(60)
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
def watch_whitelists():
|
||||
"""
|
||||
Retrieve whitelist elements from the remote URLs
|
||||
defined in config/watchers. For each (new ?) element,
|
||||
add it to the DB.
|
||||
"""
|
||||
|
||||
urls = read_config(("watchers", "whitelists"))
|
||||
watchers = [{"url": url, "status": False} for url in urls]
|
||||
|
||||
while True:
|
||||
for w in watchers:
|
||||
if w["status"] == False:
|
||||
whitelist = WhiteList()
|
||||
elements = []
|
||||
to_delete = []
|
||||
try:
|
||||
res = requests.get(w["url"], verify=True)
|
||||
if res.status_code == 200:
|
||||
content = json.loads(res.content)
|
||||
elements = content["elements"] if "elements" in content else []
|
||||
to_delete = content["to_delete"] if "to_delete" in content else []
|
||||
else:
|
||||
w["status"] = False
|
||||
except:
|
||||
w["status"] = False
|
||||
|
||||
for elem in elements:
|
||||
try:
|
||||
whitelist.add(elem["type"], elem["element"], "watcher")
|
||||
w["status"] = True
|
||||
except:
|
||||
continue
|
||||
|
||||
for elem in to_delete:
|
||||
try:
|
||||
whitelist.delete_by_value(elem["element"])
|
||||
w["status"] = True
|
||||
except:
|
||||
continue
|
||||
|
||||
if False in [w["status"] for w in watchers]:
|
||||
time.sleep(60)
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
def watch_misp():
|
||||
"""
|
||||
Retrieve IOCs from misp instances. Each new element is
|
||||
tested and then added to the database.
|
||||
"""
|
||||
iocs, misp = IOCs(), MISP()
|
||||
instances = [i for i in misp.get_instances()]
|
||||
|
||||
while instances:
|
||||
for i, ist in enumerate(instances):
|
||||
status = misp.test_instance(ist["url"],
|
||||
ist["apikey"],
|
||||
ist["verifycert"])
|
||||
if status:
|
||||
for ioc in misp.get_iocs(ist["id"]):
|
||||
iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
|
||||
ioc["value"], "misp-{}".format(ist["id"]))
|
||||
misp.update_sync(ist["id"])
|
||||
instances.pop(i)
|
||||
if instances: time.sleep(60)
|
||||
|
||||
|
||||
p1 = Process(target=watch_iocs)
|
||||
p2 = Process(target=watch_whitelists)
|
||||
p3 = Process(target=watch_misp)
|
||||
|
||||
p1.start()
|
||||
p2.start()
|
||||
p3.start()
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from app.utils import read_config
|
||||
from app.classes.iocs import IOCs
|
||||
from app.classes.whitelist import WhiteList
|
||||
from app.classes.misp import MISP
|
||||
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
import time
|
||||
from multiprocessing import Process
|
||||
|
||||
"""
|
||||
This file is parsing the watchers present
|
||||
in the configuration file. This in order to get
|
||||
automatically new iocs / elements from remote
|
||||
sources without user interaction.
|
||||
"""
|
||||
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
|
||||
def watch_iocs():
|
||||
"""
|
||||
Retrieve IOCs from the remote URLs defined in config/watchers.
|
||||
For each IOC, add it to the DB.
|
||||
"""
|
||||
|
||||
# Retrieve the URLs from the configuration
|
||||
urls = read_config(("watchers", "iocs"))
|
||||
watchers = [{"url": url, "status": False} for url in urls]
|
||||
|
||||
while True:
|
||||
for w in watchers:
|
||||
if w["status"] == False:
|
||||
iocs = IOCs()
|
||||
iocs_list = []
|
||||
to_delete = []
|
||||
try:
|
||||
res = requests.get(w["url"], verify=False)
|
||||
if res.status_code == 200:
|
||||
content = json.loads(res.content)
|
||||
iocs_list = content["iocs"] if "iocs" in content else []
|
||||
to_delete = content["to_delete"] if "to_delete" in content else []
|
||||
else:
|
||||
w["status"] = False
|
||||
except:
|
||||
w["status"] = False
|
||||
|
||||
for ioc in iocs_list:
|
||||
try:
|
||||
iocs.add(ioc["type"], ioc["tag"],
|
||||
ioc["tlp"], ioc["value"], "watcher")
|
||||
w["status"] = True
|
||||
except:
|
||||
continue
|
||||
|
||||
for ioc in to_delete:
|
||||
try:
|
||||
iocs.delete_by_value(ioc["value"])
|
||||
w["status"] = True
|
||||
except:
|
||||
continue
|
||||
|
||||
# If at least one URL haven't be parsed, let's retry in 1min.
|
||||
if False in [w["status"] for w in watchers]:
|
||||
time.sleep(60)
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
def watch_whitelists():
|
||||
"""
|
||||
Retrieve whitelist elements from the remote URLs
|
||||
defined in config/watchers. For each (new ?) element,
|
||||
add it to the DB.
|
||||
"""
|
||||
|
||||
urls = read_config(("watchers", "whitelists"))
|
||||
watchers = [{"url": url, "status": False} for url in urls]
|
||||
|
||||
while True:
|
||||
for w in watchers:
|
||||
if w["status"] == False:
|
||||
whitelist = WhiteList()
|
||||
elements = []
|
||||
to_delete = []
|
||||
try:
|
||||
res = requests.get(w["url"], verify=False)
|
||||
if res.status_code == 200:
|
||||
content = json.loads(res.content)
|
||||
elements = content["elements"] if "elements" in content else []
|
||||
to_delete = content["to_delete"] if "to_delete" in content else []
|
||||
else:
|
||||
w["status"] = False
|
||||
except:
|
||||
w["status"] = False
|
||||
|
||||
for elem in elements:
|
||||
try:
|
||||
whitelist.add(elem["type"], elem["element"], "watcher")
|
||||
w["status"] = True
|
||||
except:
|
||||
continue
|
||||
|
||||
for elem in to_delete:
|
||||
try:
|
||||
whitelist.delete_by_value(elem["element"])
|
||||
w["status"] = True
|
||||
except:
|
||||
continue
|
||||
|
||||
if False in [w["status"] for w in watchers]:
|
||||
time.sleep(60)
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
def watch_misp():
|
||||
"""
|
||||
Retrieve IOCs from misp instances. Each new element is
|
||||
tested and then added to the database.
|
||||
"""
|
||||
iocs, misp = IOCs(), MISP()
|
||||
instances = [i for i in misp.get_instances()]
|
||||
|
||||
while instances:
|
||||
for i, ist in enumerate(instances):
|
||||
status = misp.test_instance(ist["url"],
|
||||
ist["apikey"],
|
||||
ist["verifycert"])
|
||||
if status:
|
||||
for ioc in misp.get_iocs(ist["id"]):
|
||||
iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
|
||||
ioc["value"], "misp-{}".format(ist["id"]))
|
||||
misp.update_sync(ist["id"])
|
||||
instances.pop(i)
|
||||
if instances: time.sleep(60)
|
||||
|
||||
|
||||
p1 = Process(target=watch_iocs)
|
||||
p2 = Process(target=watch_whitelists)
|
||||
p3 = Process(target=watch_misp)
|
||||
|
||||
p1.start()
|
||||
p2.start()
|
||||
p3.start()
|
||||
|
||||
Reference in New Issue
Block a user