370 lines
22 KiB
Python
370 lines
22 KiB
Python
|
|
||
|
|
#!/usr/bin/env python3
|
||
|
|
# -*- coding: utf-8 -*-
|
||
|
|
|
||
|
|
from classes.parsezeeklogs import ParseZeekLogs
|
||
|
|
from netaddr import IPNetwork, IPAddress
|
||
|
|
from utils import get_iocs, get_config, get_whitelist
|
||
|
|
from ipwhois import IPWhois
|
||
|
|
import subprocess as sp
|
||
|
|
import json
|
||
|
|
import pydig
|
||
|
|
import os
|
||
|
|
|
||
|
|
|
||
|
|
class ZeekEngine(object):
|
||
|
|
|
||
|
|
def __init__(self, capture_directory):
|
||
|
|
self.working_dir = capture_directory
|
||
|
|
self.alerts = []
|
||
|
|
self.conns = []
|
||
|
|
self.ssl = []
|
||
|
|
self.http = []
|
||
|
|
self.dns = []
|
||
|
|
self.files = []
|
||
|
|
|
||
|
|
# Get analysis configuration
|
||
|
|
self.heuristics_analysis = get_config(("analysis", "heuristics"))
|
||
|
|
self.iocs_analysis = get_config(("analysis", "iocs"))
|
||
|
|
self.white_analysis = get_config(("analysis", "whitelist"))
|
||
|
|
|
||
|
|
def fill_dns(self, dir):
|
||
|
|
"""
|
||
|
|
Fill the DNS resolutions thanks to the dns.log.
|
||
|
|
:return: nothing - all resolutions appended to self.dns.
|
||
|
|
"""
|
||
|
|
if os.path.isfile(os.path.join(dir, "dns.log")):
|
||
|
|
for record in ParseZeekLogs(os.path.join(dir, "dns.log"), output_format="json", safe_headers=False):
|
||
|
|
if record is not None:
|
||
|
|
if record["qtype_name"] in ["A", "AAAA"]:
|
||
|
|
d = {"domain": record["query"],
|
||
|
|
"answers": record["answers"].split(",")}
|
||
|
|
if d not in self.dns:
|
||
|
|
self.dns.append(d)
|
||
|
|
|
||
|
|
def netflow_check(self, dir):
|
||
|
|
"""
|
||
|
|
Enrich and check the netflow from the conn.log against whitelist and IOCs.
|
||
|
|
:return: nothing - all stuff appended to self.alerts
|
||
|
|
"""
|
||
|
|
max_ports = get_config(("analysis", "max_ports"))
|
||
|
|
http_default_port = get_config(("analysis", "http_default_port"))
|
||
|
|
|
||
|
|
# Get the netflow from conn.log.
|
||
|
|
if os.path.isfile(os.path.join(dir, "conn.log")):
|
||
|
|
for record in ParseZeekLogs(os.path.join(dir, "conn.log"), output_format="json", safe_headers=False):
|
||
|
|
if record is not None:
|
||
|
|
c = {"ip_dst": record["id.resp_h"],
|
||
|
|
"proto": record["proto"],
|
||
|
|
"port_dst": record["id.resp_p"],
|
||
|
|
"service": record["service"]}
|
||
|
|
if c not in self.conns:
|
||
|
|
self.conns.append(c)
|
||
|
|
|
||
|
|
# Let's add some dns resolutions.
|
||
|
|
for c in self.conns:
|
||
|
|
c["resolution"] = self.resolve(c["ip_dst"])
|
||
|
|
|
||
|
|
# Order the conns list by the resolution field.
|
||
|
|
self.conns = sorted(self.conns, key=lambda c: c["resolution"])
|
||
|
|
|
||
|
|
# Check for whitelisted assets, if any, delete the record.
|
||
|
|
if self.white_analysis:
|
||
|
|
|
||
|
|
wl_cidrs = [IPNetwork(cidr) for cidr in get_whitelist("cidr")]
|
||
|
|
wl_hosts = get_whitelist("ip4addr") + get_whitelist("ip6addr")
|
||
|
|
wl_domains = get_whitelist("domain")
|
||
|
|
|
||
|
|
for i, c in enumerate(self.conns):
|
||
|
|
if c["ip_dst"] in [ip for ip in wl_hosts]:
|
||
|
|
self.conns[i] = False
|
||
|
|
elif c["resolution"] in wl_domains:
|
||
|
|
self.conns[i] = False
|
||
|
|
elif True in [c["resolution"].endswith("." + dom) for dom in wl_domains]:
|
||
|
|
self.conns[i] = False
|
||
|
|
elif True in [IPAddress(c["ip_dst"]) in cidr for cidr in wl_cidrs]:
|
||
|
|
self.conns[i] = False
|
||
|
|
|
||
|
|
# Let's delete whitelisted connections.
|
||
|
|
self.conns = list(filter(lambda c: c != False, self.conns))
|
||
|
|
|
||
|
|
if self.heuristics_analysis:
|
||
|
|
for c in self.conns:
|
||
|
|
# Check for UDP / ICMP (strange from a smartphone.)
|
||
|
|
if c["proto"] in ["UDP", "ICMP"]:
|
||
|
|
self.alerts.append({"title": "{} communication going outside the local network to {}.".format(c["proto"].upper(), c["resolution"]),
|
||
|
|
"description": "The {} protocol is commonly used in internal networks. Please, verify if the host {} leveraged other alerts which may ".format(c["proto"].upper(), c["resolution"])
|
||
|
|
+ "indicates a possible malicious behavior.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "PROTO-01"})
|
||
|
|
# Check for use of ports over 1024.
|
||
|
|
if c["port_dst"] >= max_ports:
|
||
|
|
self.alerts.append({"title": "{} connection to {} to a port over or equal to {}.".format(c["proto"].upper(), c["resolution"], max_ports),
|
||
|
|
"description": "{} connections have been seen to {} by using the port {}. The use of non-standard port can be sometimes associated to malicious activities. ".format(c["proto"].upper(), c["resolution"], c["port_dst"])
|
||
|
|
+ "We recommend to check if this host has a good reputation by looking on other alerts and search it on the internet.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Low",
|
||
|
|
"id": "PROTO-02"})
|
||
|
|
# Check for use of HTTP.
|
||
|
|
if c["service"] == "http" and c["port_dst"] == http_default_port:
|
||
|
|
self.alerts.append({"title": "HTTP communications been done to the host {}".format(c["resolution"]),
|
||
|
|
"description": "Your device exchanged with the host {} by using HTTP, an unencrypted protocol. ".format(c["resolution"])
|
||
|
|
+ "Even if this behavior is not malicious by itself, it is unusual to see HTTP communications issued from smartphone applications "
|
||
|
|
+ "running in the background. Please check the host reputation by searching it on the internet.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Low",
|
||
|
|
"id": "PROTO-03"})
|
||
|
|
|
||
|
|
# Check for use of HTTP on a non standard port.
|
||
|
|
if c["service"] == "http" and c["port_dst"] != http_default_port:
|
||
|
|
self.alerts.append({"title": "HTTP communications have been seen to the host {} on a non standard port ({}).".format(c["resolution"], c["port_dst"]),
|
||
|
|
"description": "Your device exchanged with the host {} by using HTTP, an unencrypted protocol on the port {}. ".format(c["resolution"], c["port_dst"])
|
||
|
|
+ "This behavior is quite unusual. Please check the host reputation by searching it on the internet.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "PROTO-04"})
|
||
|
|
# Check for non-resolved IP address.
|
||
|
|
if c["service"] == c["resolution"]:
|
||
|
|
self.alerts.append({"title": "The server {} hasn't been resolved by any DNS query during the session".format(c["ip_dst"]),
|
||
|
|
"description": "It means that the server {} is likely not resolved by any domain name or the resolution has already been cached by ".format(c["ip_dst"])
|
||
|
|
+ "the device. If the host appears in other alerts, please check it.",
|
||
|
|
"host": c["ip_dst"],
|
||
|
|
"level": "Low",
|
||
|
|
"id": "PROTO-05"})
|
||
|
|
|
||
|
|
if self.iocs_analysis:
|
||
|
|
|
||
|
|
bl_cidrs = [[IPNetwork(cidr[0]), cidr[1]]
|
||
|
|
for cidr in get_iocs("cidr")]
|
||
|
|
bl_hosts = get_iocs("ip4addr") + get_iocs("ip6addr")
|
||
|
|
bl_domains = get_iocs("domain")
|
||
|
|
bl_freedns = get_iocs("freedns")
|
||
|
|
bl_nameservers = get_iocs("ns")
|
||
|
|
bl_tlds = get_iocs("tld")
|
||
|
|
|
||
|
|
for c in self.conns:
|
||
|
|
# Check for blacklisted IP address.
|
||
|
|
for host in bl_hosts:
|
||
|
|
if c["ip_dst"] == host[0]:
|
||
|
|
self.alerts.append({"title": "A connection has been made to {} ({}) which is tagged as {}.".format(c["resolution"], c["ip_dst"], host[1].upper()),
|
||
|
|
"description": "The host {} has been explicitly blacklisted for malicious activities. Your device is likely compromised ".format(c["ip_dst"])
|
||
|
|
+ "and needs to be investigated more deeply by IT security professionals.",
|
||
|
|
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "High",
|
||
|
|
"id": "IOC-01"})
|
||
|
|
break
|
||
|
|
# Check for blacklisted CIDR.
|
||
|
|
for cidr in bl_cidrs:
|
||
|
|
if IPAddress(c["ip_dst"]) in cidr[0]:
|
||
|
|
self.alerts.append({"title": "Communication to {} under the CIDR {} which is tagged as {}.".format(c["resolution"], cidr[0], cidr[1].upper()),
|
||
|
|
"description": "The server {} is hosted under a network which is known to host malicious activities. Even if this behavior is not malicious by itself, ".format(c["resolution"])
|
||
|
|
+ "you need to check if other alerts are also mentioning this host. If you have some doubts, please "
|
||
|
|
+ "search this host on the internet to see if its legit or not.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "IOC-02"})
|
||
|
|
# Check for blacklisted domain.
|
||
|
|
for domain in bl_domains:
|
||
|
|
if c["resolution"].endswith(domain[0]):
|
||
|
|
if domain[1] != "tracker":
|
||
|
|
self.alerts.append({"title": "A DNS request have been done to {} which is tagged as {}.".format(c["resolution"], domain[1].upper()),
|
||
|
|
"description": "The domain name {} seen in the capture has been explicitly tagged as malicious. This indicates that ".format(c["resolution"])
|
||
|
|
+ "your device is likely compromised and needs to be investigated deeply.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "High",
|
||
|
|
"id": "IOC-03"})
|
||
|
|
else:
|
||
|
|
self.alerts.append({"title": "A DNS request have been done to {} which is tagged as {}.".format(c["resolution"], domain[1].upper()),
|
||
|
|
"description": "The domain name {} seen in the capture has been explicitly tagged as a Tracker. This ".format(c["resolution"])
|
||
|
|
+ "indicates that one of the active apps is geo-tracking your moves.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "IOC-03"})
|
||
|
|
# Check for blacklisted FreeDNS.
|
||
|
|
for domain in bl_freedns:
|
||
|
|
if c["resolution"].endswith("." + domain[0]):
|
||
|
|
self.alerts.append({"title": "A DNS request have been done to the domain {} which is a Free DNS.".format(c["resolution"]),
|
||
|
|
"description": "The domain name {} is using a Free DNS service. This kind of service is commonly used by cybercriminals ".format(c["resolution"])
|
||
|
|
+ "or state-sponsored threat actors during their operations.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "IOC-04"})
|
||
|
|
|
||
|
|
# Check for suspect tlds.
|
||
|
|
for tld in bl_tlds:
|
||
|
|
if c["resolution"].endswith(tld[0]):
|
||
|
|
self.alerts.append({"title": "A DNS request have been done to the domain {} which contains a suspect TLD.".format(c["resolution"]),
|
||
|
|
"description": "The domain name {} is using a suspect Top Level Domain ({}). Even not malicious, this non-generic TLD is used regularly by cybercrime ".format(c["resolution"], tld[0])
|
||
|
|
+ "or state-sponsored operations. Please check this domain by searching it on an internet search engine. If other alerts are related to this "
|
||
|
|
+ "host, please consider it as very suspicious.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Low",
|
||
|
|
"id": "IOC-05"})
|
||
|
|
|
||
|
|
# Check for use of suspect nameservers.
|
||
|
|
try:
|
||
|
|
name_servers = pydig.query(c["resolution"], "NS")
|
||
|
|
except:
|
||
|
|
name_servers = []
|
||
|
|
|
||
|
|
if len(name_servers):
|
||
|
|
for ns in bl_nameservers:
|
||
|
|
if name_servers[0].endswith(".{}.".format(ns[0])):
|
||
|
|
self.alerts.append({"title": "The domain {} is using a suspect nameserver ({}).".format(c["resolution"], name_servers[0]),
|
||
|
|
"description": "The domain name {} is using a nameserver that has been explicitly tagged to be associated to malicious activities. ".format(c["resolution"])
|
||
|
|
+ "Many cybercriminals and state-sponsored threat actors are using this kind of registrars because they allow cryptocurrencies and anonymous payments.",
|
||
|
|
"host": c["resolution"],
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "IOC-06"})
|
||
|
|
|
||
|
|
def files_check(self, dir):
|
||
|
|
"""
|
||
|
|
Check on the files.log:
|
||
|
|
* Check certificates sha1
|
||
|
|
* [todo] Check possible binary data or APKs?
|
||
|
|
:return: nothing - all stuff appended to self.alerts
|
||
|
|
"""
|
||
|
|
|
||
|
|
if not self.iocs_analysis:
|
||
|
|
return
|
||
|
|
|
||
|
|
bl_certs = get_iocs("sha1cert")
|
||
|
|
|
||
|
|
if os.path.isfile(os.path.join(dir, "files.log")):
|
||
|
|
for record in ParseZeekLogs(os.path.join(dir, "files.log"), output_format="json", safe_headers=False):
|
||
|
|
if record is not None:
|
||
|
|
f = {"filename": record["filename"],
|
||
|
|
"ip_src": record["tx_hosts"],
|
||
|
|
"ip_dst": record["rx_hosts"],
|
||
|
|
"mime_type": record["mime_type"],
|
||
|
|
"sha1": record["sha1"]}
|
||
|
|
if f not in self.files:
|
||
|
|
self.files.append(f)
|
||
|
|
|
||
|
|
for f in self.files:
|
||
|
|
if f["mime_type"] == "application/x-x509-ca-cert":
|
||
|
|
for cert in bl_certs: # Check for blacklisted certificate.
|
||
|
|
if f["sha1"] == cert[0]:
|
||
|
|
host = self.resolve(f["ip_dst"])
|
||
|
|
self.alerts.append({"title": "A certificate associated to {} activities have been found in the communication to {}.".format(cert[1].upper(), host),
|
||
|
|
"description": "The certificate ({}) associated to {} has been explicitly tagged as malicious. This indicates that ".format(f["sha1"], host)
|
||
|
|
+ "your device is likely compromised and need a forensic analysis.",
|
||
|
|
"host": host,
|
||
|
|
"level": "High",
|
||
|
|
"id": "IOC-07"})
|
||
|
|
|
||
|
|
def ssl_check(self, dir):
|
||
|
|
"""
|
||
|
|
Check on the ssl.log:
|
||
|
|
* SSL connections which doesn't use the 443.
|
||
|
|
* "Free" certificate issuer (taken from the config).
|
||
|
|
* Self-signed certificates.
|
||
|
|
:return: nothing - all stuff appended to self.alerts
|
||
|
|
"""
|
||
|
|
ssl_default_ports = get_config(("analysis", "ssl_default_ports"))
|
||
|
|
free_issuers = get_config(("analysis", "free_issuers"))
|
||
|
|
|
||
|
|
if os.path.isfile(os.path.join(dir, "ssl.log")):
|
||
|
|
for record in ParseZeekLogs(os.path.join(dir, "ssl.log"), output_format="json", safe_headers=False):
|
||
|
|
if record is not None:
|
||
|
|
c = {"host": record['id.resp_h'],
|
||
|
|
"port": record['id.resp_p'],
|
||
|
|
"issuer": record["issuer"],
|
||
|
|
"validation_status": record["validation_status"]}
|
||
|
|
if c not in self.ssl:
|
||
|
|
self.ssl.append(c)
|
||
|
|
|
||
|
|
if self.heuristics_analysis:
|
||
|
|
for cert in self.ssl:
|
||
|
|
host = self.resolve(cert["host"])
|
||
|
|
|
||
|
|
# If the associated host has not whitelisted, check the cert.
|
||
|
|
if host in [c["resolution"] for c in self.conns]:
|
||
|
|
|
||
|
|
# Check for non generic SSL port.
|
||
|
|
if cert["port"] not in ssl_default_ports:
|
||
|
|
self.alerts.append({"title": "SSL connection done on an non standart port ({}) to {}".format(cert["port"], host),
|
||
|
|
"description": "It is not common to see SSL connections issued from smartphones using non-standard ports. Even this can be totally legit,"
|
||
|
|
+ " we recommend to check the reputation of {}, by looking at its WHOIS record, the associated autonomus system, its creation date, and ".format(host)
|
||
|
|
+ " by searching it the internet.",
|
||
|
|
"host": host,
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "SSL-01"})
|
||
|
|
# Check Free SSL certificates.
|
||
|
|
if cert["issuer"] in free_issuers:
|
||
|
|
self.alerts.append({"title": "An SSL connection to {} is using a free certificate.".format(host),
|
||
|
|
"description": "Free certificates — such as Let's Encrypt — are wildly used by command and control servers associated to "
|
||
|
|
+ "malicious implants or phishing web pages. We recommend to check the host associated to this certificate, "
|
||
|
|
+ "by looking at the domain name, its creation date, or by checking its reputation on the internet.",
|
||
|
|
"host": host,
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "SSL-02"})
|
||
|
|
# Check for self-signed certificates.
|
||
|
|
if cert["validation_status"] == "self signed certificate in certificate chain":
|
||
|
|
self.alerts.append({"title": "The certificate associated to {} is self-signed.".format(host),
|
||
|
|
"description": "The use of self-signed certificates is a common thing for attacker infrastructure. We recommend to check the host {} ".format(host)
|
||
|
|
+ "which is associated to this certificate, by looking at the domain name (if any), its WHOIS record, its creation date, and "
|
||
|
|
+ " by checking its reputation on the internet.",
|
||
|
|
"host": host,
|
||
|
|
"level": "Moderate",
|
||
|
|
"id": "SSL-03"})
|
||
|
|
|
||
|
|
def alerts_check(self):
|
||
|
|
"""
|
||
|
|
Leverage an advice to the user based on the trigered hosts
|
||
|
|
:return: nothing - all generated alerts appended to self.alerts
|
||
|
|
"""
|
||
|
|
hosts = {}
|
||
|
|
|
||
|
|
for alert in [dict(t) for t in {tuple(d.items()) for d in self.alerts}]:
|
||
|
|
if alert["host"] not in hosts:
|
||
|
|
hosts[alert["host"]] = 1
|
||
|
|
else:
|
||
|
|
hosts[alert["host"]] += 1
|
||
|
|
|
||
|
|
for host, nb in hosts.items():
|
||
|
|
if nb >= get_config(("analysis", "max_alerts")):
|
||
|
|
self.alerts.append({"title": "Check alerts for {}".format(host),
|
||
|
|
"description": "Please, check the reputation of the host {}, this one seems to be malicious as it leveraged {} alerts during the session.".format(host, nb),
|
||
|
|
"host": host,
|
||
|
|
"level": "High",
|
||
|
|
"id": "ADV-01"})
|
||
|
|
|
||
|
|
def resolve(self, ip_addr):
|
||
|
|
"""
|
||
|
|
A simple method to retreive DNS names from IP addresses
|
||
|
|
in order to replace them in alerts.
|
||
|
|
|
||
|
|
:return: String - DNS record or IP Address.
|
||
|
|
"""
|
||
|
|
for record in self.dns:
|
||
|
|
if ip_addr in record["answers"]:
|
||
|
|
return record["domain"]
|
||
|
|
return ip_addr
|
||
|
|
|
||
|
|
def start_zeek(self):
|
||
|
|
"""
|
||
|
|
Start zeek and check the logs.
|
||
|
|
"""
|
||
|
|
sp.Popen("cd {} && /opt/zeek/bin/zeek -Cr capture.pcap protocols/ssl/validate-certs".format(
|
||
|
|
self.working_dir), shell=True).wait()
|
||
|
|
sp.Popen("cd {} && mkdir assets".format(
|
||
|
|
self.working_dir), shell=True).wait()
|
||
|
|
sp.Popen("cd {} && mv *.log assets/".format(self.working_dir),
|
||
|
|
shell=True).wait()
|
||
|
|
|
||
|
|
self.fill_dns(self.working_dir + "/assets/")
|
||
|
|
self.netflow_check(self.working_dir + "/assets/")
|
||
|
|
self.ssl_check(self.working_dir + "/assets/")
|
||
|
|
self.files_check(self.working_dir + "/assets/")
|
||
|
|
self.alerts_check()
|
||
|
|
|
||
|
|
def get_alerts(self):
|
||
|
|
"""
|
||
|
|
Retreive alerts.
|
||
|
|
:return: list - a list of alerts wihout duplicates.
|
||
|
|
"""
|
||
|
|
return [dict(t) for t in {tuple(d.items()) for d in self.alerts}]
|