Working on the MISP integration, still WIP (untested)

This commit is contained in:
Félix Aime 2021-06-08 18:22:52 +02:00
parent c1b8f4a447
commit e0c79fa5d6
6 changed files with 238 additions and 268 deletions

View File

@ -3,17 +3,17 @@
from flask import Blueprint, jsonify, Response, request
from app.decorators import require_header_token, require_get_token
from app.classes.mispobj import MISPObj
from app.classes.misp import MISP
import json
misp_bp = Blueprint("misp", __name__)
misp = MISPObj()
misp = MISP()
@misp_bp.route('/add', methods=['POST'])
@require_header_token
def add ():
def add():
"""
Parse and add a MISP instance to the database.
:return: status of the operation in JSON
@ -22,9 +22,11 @@ def add ():
instance = data["data"]["instance"]
source = "backend"
res = MISPObj.add(instance["name"], instance["url"], instance["key"], instance["ssl"], source)
res = MISP.add(instance["name"], instance["url"],
instance["key"], instance["ssl"], source)
return jsonify(res)
@misp_bp.route('/delete/<misp_id>', methods=['GET'])
@require_header_token
def delete(misp_id):
@ -32,9 +34,10 @@ def delete(misp_id):
Delete a MISP instance by its id to the database.
:return: status of the operation in JSON
"""
res = MISPObj.delete(misp_id)
res = MISP.delete(misp_id)
return jsonify(res)
@misp_bp.route('/get_all', methods=['GET'])
@require_header_token
def get_all():
@ -42,12 +45,12 @@ def get_all():
Retreive a list of all MISP instances.
:return: list of MISP instances in JSON.
"""
res = MISPObj.get_all()
res = MISP.get_all()
return jsonify({"results": [i for i in res]})
@misp_bp.route('/get_iocs', methods=['POST'])
#@require_header_token
# @require_header_token
def get_iocs():
"""
Retreive a list of all MISP instances.
@ -57,20 +60,24 @@ def get_iocs():
data = json.loads(request.data)
data = data["data"]
res = MISPObj.get_iocs(data["misp_id"], data["limit"], data["page"])
print(res)
res = MISP.get_iocs(data["misp_id"],
data["limit"],
data["page"])
return jsonify(res)
@misp_bp.route('/edit', methods=['POST'])
@require_header_token
def edit ():
def edit():
"""
Parse and edit the desired MISP instance.
:return: status of the operation in JSON
"""
data = json.loads(request.data)
instance = data["data"]["instance"]
print(instance)
res = MISPObj.edit(instance["id"], instance["name"], instance["url"], instance["apikey"], instance["verifycert"])
res = MISP.edit(instance["id"],
instance["name"],
instance["url"],
instance["apikey"],
instance["verifycert"])
return jsonify(res)

View File

@ -0,0 +1,181 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from app import db
from app.db.models import MISPInst
from sqlalchemy.sql import exists
from app.definitions import definitions as defs
from urllib.parse import unquote
from flask import escape
from pymisp import PyMISP
import re
import time
import sys
class MISP(object):
def __init__(self):
return None
@staticmethod
def add(misp_name, misp_url, misp_key, misp_verifycert):
"""
Parse and add a MISP instance to the database.
:return: status of the operation in JSON
"""
sameinstances = db.session.query(MISPInst).filter(
MISPInst.url == misp_url, MISPInst.apikey == misp_key)
if sameinstances.count():
return {"status": False,
"message": "This MISP instance already exists"}
elif misp_name != "":
if misp_url != "":
if re.match(r"^(?:(?:http|https)://)", misp_url):
if misp_key != "":
added_on = int(time.time())
db.session.add(MISPInst(misp_name, escape(
misp_url), misp_key, misp_verifycert, added_on))
db.session.commit()
return {"status": True,
"message": "MISP instance added",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The key can't be empty"}
else:
return {"status": False,
"message": "The url must begin with http:// or https://"}
else:
return {"status": False,
"message": "The url can't be empty"}
else:
return {"status": False,
"message": "The MISP instance name can't be empty"}
@staticmethod
def edit(misp_id, misp_name, misp_url, misp_key, misp_verifycert):
"""
Parse and edit the desired MISP instance.
:return: status of the operation in JSON
"""
misp = MISPInst.query.get(int(misp_id))
otherinstances = db.session.query(MISPInst).filter(MISPInst.id != int(
misp_id), MISPInst.url == misp_url, MISPInst.apikey == misp_key)
if misp is None:
return {"status": False,
"message": "Can't find the MISP instance"}
if otherinstances.count() > 0:
return {"status": False,
"message": "This MISP instance already exists"}
elif misp_name != "":
if misp_url != "":
if re.match(r"^(?:(?:http|https)://)", misp_url):
if misp_key != "":
misp.name = misp_name
misp.url = misp_url
misp.apikey = misp_key
misp.verifycert = misp_verifycert
db.session.commit()
return {"status": True,
"message": "MISP instance edited"}
else:
return {"status": False,
"message": "The key can't be empty"}
else:
return {"status": False,
"message": "The url must begin with http:// or https://"}
else:
return {"status": False,
"message": "The url can't be empty"}
else:
return {"status": False,
"message": "The MISP instance name can't be empty"}
@staticmethod
def delete_instance(misp_id):
"""
Delete a MISP instance by its id in the database.
:return: status of the operation in JSON
"""
if db.session.query(exists().where(MISPInst.id == misp_id)).scalar():
db.session.query(MISPInst).filter_by(id=misp_id).delete()
db.session.commit()
return {"status": True,
"message": "MISP instance deleted"}
else:
return {"status": False,
"message": "MISP instance not found"}
@staticmethod
def get_instances():
"""
Get MISP instances from the database
:return: generator of the records.
"""
for misp in db.session.query(MISPInst).all():
misp = misp.__dict__
yield {"id": misp["id"],
"name": misp["name"],
"url": misp["url"],
"apikey": misp["apikey"],
"verifycert": misp["verifycert"]}
@staticmethod
def get_iocs(misp_id):
"""
Get all IOCs from specific MISP instance
/!\ Todo: NEED TO ADD LAST SYNCHRO DATE + page etc. stuff.
:return: generator containing the IOCs.
"""
misp = MISPInst.query.get(int(misp_id))
if misp is not None:
if misp.url and misp.apikey:
try:
# Connect to MISP instance and get network activity attributes.
m = PyMISP(misp.url, misp.apikey, misp.verifycert)
r = m.search("attributes", category="Network activity")
for attr in r["Attribute"]:
if attr["type"] in ["ip_dst", "domain", "snort", "x509-fingerprint-sha1"]:
ioc = {"value": attr["value"],
"type": None,
"tag": "suspect",
"tlp": "white"}
# Deduce the IOC type.
if re.match(defs["iocs_types"][0]["regex"], attr["value"]):
ioc["type"] = "ipv4addr"
elif re.match(defs["iocs_types"][1]["regex"], attr["value"]):
ioc["type"] = "ipv6addr"
elif re.match(defs["iocs_types"][3]["regex"], attr["value"]):
ioc["type"] = "domain"
elif re.match(defs["iocs_types"][4]["regex"], attr["value"]):
ioc["type"] = "sha1cert"
elif "alert " in attr["value"][0:5]:
ioc["type"] = "snort"
if "Tag" in attr:
for tag in attribute['Tag']:
# Add the TLP of the IOC.
tlp = re.search(r"^(?:tlp:)(red|green|amber|white)", tag['name'])
if tlp: ioc["tlp"] = tlp.group(1)
# Add possible tag.
if lower(tag["name"]) in [t["tag"] for t in defs["iocs_tags"]]:
ioc["tag"] = lower(tag["name"])
yield ioc
except:
return {"status": False,
"message": "An exception has been raised: ", sys.exc_info()[0])}
pass
else:
return {"status": False,
"message": "The URL or API key is empty."}
else:
return {"status": False,
"message": "Unknown MISP instance."}

View File

@ -1,240 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from app import db
from app.db.models import MISPInst
from sqlalchemy.sql import exists
from app.definitions import definitions
from urllib.parse import unquote
from flask import escape
from pymisp import PyMISP
import re
import time
import sys
class MISPObj(object):
def __init__(self):
return None
@staticmethod
def add(misp_name, misp_url, misp_key, misp_verifycert, source):
"""
Parse and add a MISP"instance to the database.
:return: status of the operation in JSON
"""
sameinstances = db.session.query(MISPInst).filter(MISPInst.url == misp_url, MISPInst.apikey == misp_key)
if sameinstances.count() > 0:
return {"status": False,
"message": "This MISP instance already exists",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
elif misp_name != "":
if misp_url != "":
if re.match(r"^(?:(?:http|https)://)", misp_url):
if misp_key != "":
added_on = int(time.time())
db.session.add(MISPInst(misp_name, escape(misp_url), misp_key, misp_verifycert, source, added_on))
db.session.commit()
return {"status": True,
"message": "MISP instance added",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The key can't be empty",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": "",
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The url must begin with http:// or https://",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The url can't be empty",
"name": escape(misp_name),
"url": "",
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The MISP instance name can't be empty",
"name":"",
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
@staticmethod
def edit(misp_id, misp_name, misp_url, misp_key, misp_verifycert):
"""
Parse and edit the desired MISP instance.
:return: status of the operation in JSON
"""
mispinstance = MISPInst.query.get(int(misp_id))
otherinstances = db.session.query(MISPInst).filter(MISPInst.id != int(misp_id), MISPInst.url == misp_url, MISPInst.apikey == misp_key)
if mispinstance is None:
return {"status": False,
"message": "Can't find the MISP instance"}
if otherinstances.count() > 0:
return {"status": False,
"message": "This MISP instance already exists",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
elif misp_name != "":
if misp_url != "":
if re.match(r"^(?:(?:http|https)://)", misp_url):
if misp_key != "":
mispinstance.name = misp_name
mispinstance.url = misp_url
mispinstance.apikey = misp_key
mispinstance.verifycert = misp_verifycert
db.session.commit()
return {"status": True,
"message": "MISP instance edited",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The key can't be empty",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": "",
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The url must begin with http:// or https://",
"name": escape(misp_name),
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The url can't be empty",
"name": escape(misp_name),
"url": "",
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
else:
return {"status": False,
"message": "The MISP instance name can't be empty",
"name":"",
"url": escape(misp_url),
"apikey": escape(misp_key),
"verifycert": escape(misp_verifycert)}
@staticmethod
def delete(misp_id):
"""
Delete a MISP instance by its id in the database.
:return: status of the operation in JSON
"""
if db.session.query(exists().where(MISPInst.id == misp_id)).scalar():
db.session.query(MISPInst).filter_by(id=misp_id).delete()
db.session.commit()
return {"status": True,
"message": "MISP instance deleted"}
else:
return {"status": False,
"message": "MISP instance not found"}
@staticmethod
def get_all():
"""
Get all MISP instances from the database
:return: generator of the records.
"""
for mispinstance in db.session.query(MISPInst).all():
mispinstance = mispinstance.__dict__
yield {"id": mispinstance["id"],
"name": mispinstance["name"],
"url": mispinstance["url"],
"apikey": mispinstance["apikey"],
"verifycert": mispinstance["verifycert"]}
@staticmethod
def get_iocs(misp_id, limit, page):
"""
Get all IOCs from the desired MISP instance
:return: generator of the records.
"""
mispinstance = MISPInst.query.get(int(misp_id))
if mispinstance is not None:
if mispinstance.url != "":
if mispinstance.apikey != "":
try:
# Connects to the desired MISP instance
mispinstance = PyMISP(mispinstance.url, mispinstance.apikey, mispinstance.verifycert)
# Retreives the attributes (or IOCs) that are supported by Tinycheck
attributes = mispinstance.search('attributes', category='Network activity', limit=limit, page=page, metadata=True)
if 'Attribute' in attributes:
iocs = []
for attribute in attributes['Attribute']:
#print(attribute)
if 'value' in attribute and attribute['value'] != '':
# We have a valid value
ioc_value = attribute['value']
ioc_type = "unknown"
ioc_tag = "No tag"
ioc_tlp = "white"
isFirstTag = True
if 'Tag' in attribute:
# We have some tags
#print (attribute['Tag'])
for tag in attribute['Tag']:
tlp = re.search(r"^(?:tlp:)(red|green|amber|white)", tag['name'])
if tlp:
# The current tag is the tlp level
ioc_tlp = tlp.group(1)
#print(ioc_tlp)
elif isFirstTag:
# It is the first retreived tag that is not a tlp
isFirstTag = False
ioc_tag = tag['name']
else:
# It is another tag and not the first one
ioc_tag += ", " + tag['name']
ioc = { "value": ioc_value,
"type": ioc_type,
"tag": ioc_tag,
"tlp": ioc_tlp }
iocs.append(ioc)
return { "status":True,
"results": iocs}
else:
return { "status":False,
"message":"No valid IOCs found."}
except TypeError as error:
print (error)
pass
except:
print("An exception has been raised: ", sys.exc_info()[0])
pass
else:
{"status": False,
"message": "The api key can't be empty"}
else:
return {"status": False,
"message": "The url can't be empty"}
else:
return {"status": False,
"message": "Unknown MISP instance."}

View File

@ -1,5 +1,6 @@
from app import db
class Ioc(db.Model):
def __init__(self, value, type, tlp, tag, source, added_on):
self.value = value
@ -9,6 +10,7 @@ class Ioc(db.Model):
self.source = source
self.added_on = added_on
class Whitelist(db.Model):
def __init__(self, element, type, source, added_on):
self.element = element
@ -16,15 +18,16 @@ class Whitelist(db.Model):
self.source = source
self.added_on = added_on
class MISPInst(db.Model):
def __init__(self, name, url, key, ssl, source, added_on):
def __init__(self, name, url, key, ssl, added_on):
self.name = name
self.url = url
self.apikey = key
self.authkey = key
self.verifycert = ssl
self.source = source
self.added_on = added_on
db.mapper(Whitelist, db.Table('whitelist', db.metadata, autoload=True))
db.mapper(Ioc, db.Table('iocs', db.metadata, autoload=True))
db.mapper(MISPInst, db.Table('mispinstance', db.metadata, autoload=True))
db.mapper(MISP, db.Table('misp', db.metadata, autoload=True))

View File

@ -6,6 +6,7 @@ from app.decorators import auth
from app.blueprints.ioc import ioc_bp
from app.blueprints.whitelist import whitelist_bp
from app.blueprints.config import config_bp
from app.blueprints.misp import misp_bp
import datetime
import secrets
import jwt
@ -56,6 +57,7 @@ def page_not_found(e):
app.register_blueprint(ioc_bp, url_prefix='/api/ioc')
app.register_blueprint(whitelist_bp, url_prefix='/api/whitelist')
app.register_blueprint(config_bp, url_prefix='/api/config')
app.register_blueprint(misp_bp, url_prefix='/api/misp')
if __name__ == '__main__':
ssl_cert = "{}/{}".format(path[0], 'cert.pem')

View File

@ -4,6 +4,7 @@
from app.utils import read_config
from app.classes.iocs import IOCs
from app.classes.whitelist import WhiteList
from app.classes.misp import MISP
import requests
import json
@ -16,11 +17,6 @@ from multiprocessing import Process
in the configuration file. This in order to get
automatically new iocs / elements from remote
sources without user interaction.
As of today the default export JSON format from
the backend and unauthenticated HTTP requests
are accepted. The code is little awkward, it'll
be better in a next version ;)
"""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@ -29,7 +25,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def watch_iocs():
"""
Retrieve IOCs from the remote URLs defined in config/watchers.
For each (new ?) IOC, add it to the DB.
For each IOC, add it to the DB.
"""
# Retrieve the URLs from the configuration
@ -45,8 +41,10 @@ def watch_iocs():
res = requests.get(w["url"], verify=False)
if res.status_code == 200:
content = json.loads(res.content)
iocs_list = content["iocs"] if "iocs" in content else []
to_delete = content["to_delete"] if "to_delete" in content else []
iocs_list = content["iocs"] if "iocs" in content else [
]
to_delete = content["to_delete"] if "to_delete" in content else [
]
else:
w["status"] = False
except:
@ -93,8 +91,10 @@ def watch_whitelists():
res = requests.get(w["url"], verify=False)
if res.status_code == 200:
content = json.loads(res.content)
elements = content["elements"] if "elements" in content else []
to_delete = content["to_delete"] if "to_delete" in content else []
elements = content["elements"] if "elements" in content else [
]
to_delete = content["to_delete"] if "to_delete" in content else [
]
else:
w["status"] = False
except:
@ -120,8 +120,25 @@ def watch_whitelists():
break
def watch_misp():
"""
Retrieve IOCs from misp instances. Each new element is
tested added to the database.
"""
while True:
for misp in MISP.get_instances():
try:
for ioc in MISP.get_iocs(misp.id):
iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
ioc["value"], "misp-{}".format(misp["name"]))
except:
continue
p1 = Process(target=watch_iocs)
p2 = Process(target=watch_whitelists)
p3 = Process(target=watch_misp)
p1.start()
p2.start()
p3.start()