c_meyer/KasperskyLab-TinyCheck
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #73 from KasperskyLab/misp

Browse Source 
Adding MISP support to dev
This commit is contained in:
Félix Aimé
2021-06-14 16:07:03 +02:00
committed by GitHub
parent 920bd6785c 6f5fd11a26
commit 793a97b530
16 changed files with 609 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
server/backend/app/blueprints/ioc.py
View File

@ -26,6 +26,20 @@ def add(ioc_type, ioc_tag, ioc_tlp, ioc_value):
return jsonify(res)
@ioc_bp.route('/add_post', methods=['POST'])
@require_header_token
def add_post():
"""
Parse and add an IOC to the database using the post method.
:return: status of the operation in JSON
"""
data = json.loads(request.data)
ioc = data["data"]["ioc"]
res = IOCs.add(ioc["ioc_type"], ioc["ioc_tag"], ioc["ioc_tlp"], ioc["ioc_value"], ioc["ioc_source"])
return jsonify(res)
@ioc_bp.route('/delete/<ioc_id>', methods=['GET'])
@require_header_token
def delete(ioc_id):

44
server/backend/app/blueprints/misp.py Normal file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from flask import Blueprint, jsonify, Response, request
from app.decorators import require_header_token, require_get_token
from app.classes.misp import MISP
import json
misp_bp = Blueprint("misp", __name__)
misp = MISP()
@misp_bp.route('/add', methods=['POST'])
@require_header_token
def add_instance():
"""
Parse and add a MISP instance to the database.
:return: status of the operation in JSON
"""
data = json.loads(request.data)
res = misp.add_instance(data["data"]["instance"])
return jsonify(res)
@misp_bp.route('/delete/<misp_id>', methods=['GET'])
@require_header_token
def delete_instance(misp_id):
"""
Delete a MISP instance by its id to the database.
:return: status of the operation in JSON
"""
res = misp.delete_instance(misp_id)
return jsonify(res)
@misp_bp.route('/get_all', methods=['GET'])
@require_header_token
def get_all():
"""
Retreive a list of all MISP instances.
:return: list of MISP instances in JSON.
"""
res = misp.get_instances()
return jsonify({"results": [i for i in res]})

10
server/backend/app/classes/iocs.py
View File

@ -56,7 +56,12 @@ class IOCs(object):
db.session.commit()
return {"status": True,
"message": "IOC added",
"ioc": escape(ioc_value)}
"ioc": escape(ioc_value),
"type": escape(ioc_type),
"tlp": escape(ioc_tlp),
"tag": escape(ioc_tag),
"source": escape(source),
"added_on": escape(added_on)}
else:
return {"status": False,
"message": "Wrong IOC format",
@ -111,7 +116,8 @@ class IOCs(object):
"type": ioc["type"],
"tag": ioc["tag"],
"tlp": ioc["tlp"],
"value": ioc["value"]}
"value": ioc["value"],
"source": ioc["source"]}
@staticmethod
def get_types():

159
server/backend/app/classes/misp.py Normal file
View File

@ -0,0 +1,159 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from app import db
from app.db.models import MISPInst
from app.definitions import definitions as defs
from sqlalchemy.sql import exists
from urllib.parse import unquote
from flask import escape
from pymisp import PyMISP
import re
import time
import sys
class MISP(object):
def __init__(self):
return None
def add_instance(self, instance):
"""
Parse and add a MISP instance to the database.
:return: status of the operation in JSON
"""
url = instance["url"]
name = instance["name"]
apikey = instance["key"]
verify = instance["ssl"]
last_sync = int(time.time()-31536000) # One year
sameinstances = db.session.query(MISPInst).filter(
MISPInst.url == url, MISPInst.apikey == apikey)
if sameinstances.count():
return {"status": False,
"message": "This MISP instance already exists"}
if name:
if self.test_instance(url, apikey, verify):
added_on = int(time.time())
db.session.add(MISPInst(name, escape(
url), apikey, verify, added_on, last_sync))
db.session.commit()
return {"status": True,
"message": "MISP instance added"}
else:
return {"status": False,
"message": "Please verify the connection to the MISP instance"}
else:
return {"status": False,
"message": "Please provide a name for your instance"}
@staticmethod
def delete_instance(misp_id):
"""
Delete a MISP instance by its id in the database.
:return: status of the operation in JSON
"""
if db.session.query(exists().where(MISPInst.id == misp_id)).scalar():
db.session.query(MISPInst).filter_by(id=misp_id).delete()
db.session.commit()
return {"status": True,
"message": "MISP instance deleted"}
else:
return {"status": False,
"message": "MISP instance not found"}
def get_instances(self):
"""
Get MISP instances from the database
:return: generator of the records.
"""
for misp in db.session.query(MISPInst).all():
misp = misp.__dict__
yield {"id": misp["id"],
"name": misp["name"],
"url": misp["url"],
"apikey": misp["apikey"],
"verifycert": True if misp["verifycert"] else False,
"connected": self.test_instance(misp["url"], misp["apikey"], misp["verifycert"]),
"lastsync": misp["last_sync"]}
@staticmethod
def test_instance(url, apikey, verify):
"""
Test the connection of the MISP instance.
:return: generator of the records.
"""
try:
PyMISP(url, apikey, verify)
return True
except:
return False
@staticmethod
def update_sync(misp_id):
"""
Update the last synchronization date by the actual date.
:return: bool, True if updated.
"""
try:
misp = MISPInst.query.get(int(misp_id))
misp.last_sync = int(time.time())
db.session.commit()
return True
except:
return False
@staticmethod
def get_iocs(misp_id):
"""
Get all IOCs from specific MISP instance
:return: generator containing the IOCs.
"""
misp = MISPInst.query.get(int(misp_id))
if misp is not None:
if misp.url and misp.apikey:
try:
# Connect to MISP instance and get network activity attributes.
m = PyMISP(misp.url, misp.apikey, misp.verifycert)
r = m.search("attributes", category="Network activity", date_from=int(misp.last_sync))
except:
print("Unable to connect to the MISP instance ({}/{}).".format(misp.url, misp.apikey))
return []
for attr in r["Attribute"]:
if attr["type"] in ["ip-dst", "domain", "snort", "x509-fingerprint-sha1"]:
ioc = {"value": attr["value"],
"type": None,
"tag": "suspect",
"tlp": "white"}
# Deduce the IOC type.
if re.match(defs["iocs_types"][0]["regex"], attr["value"]):
ioc["type"] = "ip4addr"
elif re.match(defs["iocs_types"][1]["regex"], attr["value"]):
ioc["type"] = "ip6addr"
elif re.match(defs["iocs_types"][2]["regex"], attr["value"]):
ioc["type"] = "cidr"
elif re.match(defs["iocs_types"][3]["regex"], attr["value"]):
ioc["type"] = "domain"
elif re.match(defs["iocs_types"][4]["regex"], attr["value"]):
ioc["type"] = "sha1cert"
elif "alert " in attr["value"][0:6]:
ioc["type"] = "snort"
else:
continue
if "Tag" in attr:
for tag in attr["Tag"]:
# Add a TLP to the IOC if defined in tags.
tlp = re.search(r"^(?:tlp:)(red|green|amber|white)", tag['name'].lower())
if tlp: ioc["tlp"] = tlp.group(1)
# Add possible tag (need to match TinyCheck tags)
if tag["name"].lower() in [t["tag"] for t in defs["iocs_tags"]]:
ioc["tag"] = tag["name"].lower()
yield ioc

14
server/backend/app/db/models.py
View File

@ -1,5 +1,6 @@
from app import db
class Ioc(db.Model):
def __init__(self, value, type, tlp, tag, source, added_on):
self.value = value
@ -9,6 +10,7 @@ class Ioc(db.Model):
self.source = source
self.added_on = added_on
class Whitelist(db.Model):
def __init__(self, element, type, source, added_on):
self.element = element
@ -16,5 +18,17 @@ class Whitelist(db.Model):
self.source = source
self.added_on = added_on
class MISPInst(db.Model):
def __init__(self, name, url, key, ssl, added_on, last_sync):
self.name = name
self.url = url
self.apikey = key
self.verifycert = ssl
self.added_on = added_on
self.last_sync = last_sync
db.mapper(Whitelist, db.Table('whitelist', db.metadata, autoload=True))
db.mapper(Ioc, db.Table('iocs', db.metadata, autoload=True))
db.mapper(MISPInst, db.Table('misp', db.metadata, autoload=True))

2
server/backend/main.py
View File

@ -6,6 +6,7 @@ from app.decorators import auth
from app.blueprints.ioc import ioc_bp
from app.blueprints.whitelist import whitelist_bp
from app.blueprints.config import config_bp
from app.blueprints.misp import misp_bp
import datetime
import secrets
import jwt
@ -56,6 +57,7 @@ def page_not_found(e):
app.register_blueprint(ioc_bp, url_prefix='/api/ioc')
app.register_blueprint(whitelist_bp, url_prefix='/api/whitelist')
app.register_blueprint(config_bp, url_prefix='/api/config')
app.register_blueprint(misp_bp, url_prefix='/api/misp')
if __name__ == '__main__':
ssl_cert = "{}/{}".format(path[0], 'cert.pem')

32
server/backend/watchers.py
View File

@ -4,6 +4,7 @@
from app.utils import read_config
from app.classes.iocs import IOCs
from app.classes.whitelist import WhiteList
from app.classes.misp import MISP
import requests
import json
@ -16,11 +17,6 @@ from multiprocessing import Process
in the configuration file. This in order to get
automatically new iocs / elements from remote
sources without user interaction.
As of today the default export JSON format from
the backend and unauthenticated HTTP requests
are accepted. The code is little awkward, it'll
be better in a next version ;)
"""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@ -29,7 +25,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def watch_iocs():
"""
Retrieve IOCs from the remote URLs defined in config/watchers.
For each (new ?) IOC, add it to the DB.
For each IOC, add it to the DB.
"""
# Retrieve the URLs from the configuration
@ -120,8 +116,32 @@ def watch_whitelists():
break
def watch_misp():
"""
Retrieve IOCs from misp instances. Each new element is
tested and then added to the database.
"""
iocs, misp = IOCs(), MISP()
instances = [i for i in misp.get_instances()]
while instances:
for i, ist in enumerate(instances):
status = misp.test_instance(ist["url"],
ist["apikey"],
ist["verifycert"])
if status:
for ioc in misp.get_iocs(ist["id"]):
iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
ioc["value"], "misp-{}".format(ist["id"]))
misp.update_sync(ist["id"])
instances.pop(i)
if instances: time.sleep(60)
p1 = Process(target=watch_iocs)
p2 = Process(target=watch_whitelists)
p3 = Process(target=watch_misp)
p1.start()
p2.start()
p3.start()