diff --git a/server/backend/app/blueprints/misp.py b/server/backend/app/blueprints/misp.py index 41a61ba..2f627fc 100644 --- a/server/backend/app/blueprints/misp.py +++ b/server/backend/app/blueprints/misp.py @@ -3,28 +3,30 @@ from flask import Blueprint, jsonify, Response, request from app.decorators import require_header_token, require_get_token -from app.classes.mispobj import MISPObj +from app.classes.misp import MISP import json misp_bp = Blueprint("misp", __name__) -misp = MISPObj() +misp = MISP() @misp_bp.route('/add', methods=['POST']) @require_header_token -def add (): +def add(): """ Parse and add a MISP instance to the database. :return: status of the operation in JSON """ data = json.loads(request.data) instance = data["data"]["instance"] - + source = "backend" - res = MISPObj.add(instance["name"], instance["url"], instance["key"], instance["ssl"], source) + res = MISP.add(instance["name"], instance["url"], + instance["key"], instance["ssl"], source) return jsonify(res) + @misp_bp.route('/delete/', methods=['GET']) @require_header_token def delete(misp_id): @@ -32,9 +34,10 @@ def delete(misp_id): Delete a MISP instance by its id to the database. :return: status of the operation in JSON """ - res = MISPObj.delete(misp_id) + res = MISP.delete(misp_id) return jsonify(res) + @misp_bp.route('/get_all', methods=['GET']) @require_header_token def get_all(): @@ -42,12 +45,12 @@ def get_all(): Retreive a list of all MISP instances. :return: list of MISP instances in JSON. """ - res = MISPObj.get_all() + res = MISP.get_all() return jsonify({"results": [i for i in res]}) @misp_bp.route('/get_iocs', methods=['POST']) -#@require_header_token +# @require_header_token def get_iocs(): """ Retreive a list of all MISP instances. @@ -57,20 +60,24 @@ def get_iocs(): data = json.loads(request.data) data = data["data"] - res = MISPObj.get_iocs(data["misp_id"], data["limit"], data["page"]) - print(res) + res = MISP.get_iocs(data["misp_id"], + data["limit"], + data["page"]) return jsonify(res) @misp_bp.route('/edit', methods=['POST']) @require_header_token -def edit (): +def edit(): """ Parse and edit the desired MISP instance. :return: status of the operation in JSON """ data = json.loads(request.data) instance = data["data"]["instance"] - print(instance) - res = MISPObj.edit(instance["id"], instance["name"], instance["url"], instance["apikey"], instance["verifycert"]) - return jsonify(res) \ No newline at end of file + res = MISP.edit(instance["id"], + instance["name"], + instance["url"], + instance["apikey"], + instance["verifycert"]) + return jsonify(res) diff --git a/server/backend/app/classes/misp.py b/server/backend/app/classes/misp.py new file mode 100644 index 0000000..4cf3cce --- /dev/null +++ b/server/backend/app/classes/misp.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from app import db +from app.db.models import MISPInst +from sqlalchemy.sql import exists +from app.definitions import definitions as defs +from urllib.parse import unquote +from flask import escape +from pymisp import PyMISP +import re +import time +import sys + + +class MISP(object): + def __init__(self): + return None + + @staticmethod + def add(misp_name, misp_url, misp_key, misp_verifycert): + """ + Parse and add a MISP instance to the database. + :return: status of the operation in JSON + """ + + sameinstances = db.session.query(MISPInst).filter( + MISPInst.url == misp_url, MISPInst.apikey == misp_key) + if sameinstances.count(): + return {"status": False, + "message": "This MISP instance already exists"} + elif misp_name != "": + if misp_url != "": + if re.match(r"^(?:(?:http|https)://)", misp_url): + if misp_key != "": + added_on = int(time.time()) + db.session.add(MISPInst(misp_name, escape( + misp_url), misp_key, misp_verifycert, added_on)) + db.session.commit() + return {"status": True, + "message": "MISP instance added", + "name": escape(misp_name), + "url": escape(misp_url), + "apikey": escape(misp_key), + "verifycert": escape(misp_verifycert)} + else: + return {"status": False, + "message": "The key can't be empty"} + else: + return {"status": False, + "message": "The url must begin with http:// or https://"} + else: + return {"status": False, + "message": "The url can't be empty"} + else: + return {"status": False, + "message": "The MISP instance name can't be empty"} + + @staticmethod + def edit(misp_id, misp_name, misp_url, misp_key, misp_verifycert): + """ + Parse and edit the desired MISP instance. + :return: status of the operation in JSON + """ + misp = MISPInst.query.get(int(misp_id)) + otherinstances = db.session.query(MISPInst).filter(MISPInst.id != int( + misp_id), MISPInst.url == misp_url, MISPInst.apikey == misp_key) + if misp is None: + return {"status": False, + "message": "Can't find the MISP instance"} + if otherinstances.count() > 0: + return {"status": False, + "message": "This MISP instance already exists"} + elif misp_name != "": + if misp_url != "": + if re.match(r"^(?:(?:http|https)://)", misp_url): + if misp_key != "": + misp.name = misp_name + misp.url = misp_url + misp.apikey = misp_key + misp.verifycert = misp_verifycert + db.session.commit() + return {"status": True, + "message": "MISP instance edited"} + else: + return {"status": False, + "message": "The key can't be empty"} + else: + return {"status": False, + "message": "The url must begin with http:// or https://"} + else: + return {"status": False, + "message": "The url can't be empty"} + else: + return {"status": False, + "message": "The MISP instance name can't be empty"} + + @staticmethod + def delete_instance(misp_id): + """ + Delete a MISP instance by its id in the database. + :return: status of the operation in JSON + """ + if db.session.query(exists().where(MISPInst.id == misp_id)).scalar(): + db.session.query(MISPInst).filter_by(id=misp_id).delete() + db.session.commit() + return {"status": True, + "message": "MISP instance deleted"} + else: + return {"status": False, + "message": "MISP instance not found"} + + @staticmethod + def get_instances(): + """ + Get MISP instances from the database + :return: generator of the records. + """ + for misp in db.session.query(MISPInst).all(): + misp = misp.__dict__ + yield {"id": misp["id"], + "name": misp["name"], + "url": misp["url"], + "apikey": misp["apikey"], + "verifycert": misp["verifycert"]} + + @staticmethod + def get_iocs(misp_id): + """ + Get all IOCs from specific MISP instance + /!\ Todo: NEED TO ADD LAST SYNCHRO DATE + page etc. stuff. + :return: generator containing the IOCs. + """ + misp = MISPInst.query.get(int(misp_id)) + if misp is not None: + if misp.url and misp.apikey: + try: + # Connect to MISP instance and get network activity attributes. + m = PyMISP(misp.url, misp.apikey, misp.verifycert) + r = m.search("attributes", category="Network activity") + + for attr in r["Attribute"]: + if attr["type"] in ["ip_dst", "domain", "snort", "x509-fingerprint-sha1"]: + + ioc = {"value": attr["value"], + "type": None, + "tag": "suspect", + "tlp": "white"} + + # Deduce the IOC type. + if re.match(defs["iocs_types"][0]["regex"], attr["value"]): + ioc["type"] = "ipv4addr" + elif re.match(defs["iocs_types"][1]["regex"], attr["value"]): + ioc["type"] = "ipv6addr" + elif re.match(defs["iocs_types"][3]["regex"], attr["value"]): + ioc["type"] = "domain" + elif re.match(defs["iocs_types"][4]["regex"], attr["value"]): + ioc["type"] = "sha1cert" + elif "alert " in attr["value"][0:5]: + ioc["type"] = "snort" + + if "Tag" in attr: + for tag in attribute['Tag']: + # Add the TLP of the IOC. + tlp = re.search(r"^(?:tlp:)(red|green|amber|white)", tag['name']) + if tlp: ioc["tlp"] = tlp.group(1) + + # Add possible tag. + if lower(tag["name"]) in [t["tag"] for t in defs["iocs_tags"]]: + ioc["tag"] = lower(tag["name"]) + yield ioc + except: + return {"status": False, + "message": "An exception has been raised: ", sys.exc_info()[0])} + pass + else: + return {"status": False, + "message": "The URL or API key is empty."} + else: + return {"status": False, + "message": "Unknown MISP instance."} diff --git a/server/backend/app/classes/mispobj.py b/server/backend/app/classes/mispobj.py deleted file mode 100644 index b3e22ec..0000000 --- a/server/backend/app/classes/mispobj.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -from app import db -from app.db.models import MISPInst -from sqlalchemy.sql import exists -from app.definitions import definitions -from urllib.parse import unquote -from flask import escape -from pymisp import PyMISP -import re -import time -import sys - - -class MISPObj(object): - def __init__(self): - return None - - @staticmethod - def add(misp_name, misp_url, misp_key, misp_verifycert, source): - """ - Parse and add a MISP"instance to the database. - :return: status of the operation in JSON - """ - - sameinstances = db.session.query(MISPInst).filter(MISPInst.url == misp_url, MISPInst.apikey == misp_key) - if sameinstances.count() > 0: - return {"status": False, - "message": "This MISP instance already exists", - "name": escape(misp_name), - "url": escape(misp_url), - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - elif misp_name != "": - if misp_url != "": - if re.match(r"^(?:(?:http|https)://)", misp_url): - if misp_key != "": - added_on = int(time.time()) - db.session.add(MISPInst(misp_name, escape(misp_url), misp_key, misp_verifycert, source, added_on)) - db.session.commit() - return {"status": True, - "message": "MISP instance added", - "name": escape(misp_name), - "url": escape(misp_url), - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - else: - return {"status": False, - "message": "The key can't be empty", - "name": escape(misp_name), - "url": escape(misp_url), - "apikey": "", - "verifycert": escape(misp_verifycert)} - else: - return {"status": False, - "message": "The url must begin with http:// or https://", - "name": escape(misp_name), - "url": escape(misp_url), - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - else: - return {"status": False, - "message": "The url can't be empty", - "name": escape(misp_name), - "url": "", - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - else: - return {"status": False, - "message": "The MISP instance name can't be empty", - "name":"", - "url": escape(misp_url), - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - - @staticmethod - def edit(misp_id, misp_name, misp_url, misp_key, misp_verifycert): - """ - Parse and edit the desired MISP instance. - :return: status of the operation in JSON - """ - mispinstance = MISPInst.query.get(int(misp_id)) - otherinstances = db.session.query(MISPInst).filter(MISPInst.id != int(misp_id), MISPInst.url == misp_url, MISPInst.apikey == misp_key) - if mispinstance is None: - return {"status": False, - "message": "Can't find the MISP instance"} - if otherinstances.count() > 0: - return {"status": False, - "message": "This MISP instance already exists", - "name": escape(misp_name), - "url": escape(misp_url), - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - elif misp_name != "": - if misp_url != "": - if re.match(r"^(?:(?:http|https)://)", misp_url): - if misp_key != "": - mispinstance.name = misp_name - mispinstance.url = misp_url - mispinstance.apikey = misp_key - mispinstance.verifycert = misp_verifycert - db.session.commit() - return {"status": True, - "message": "MISP instance edited", - "name": escape(misp_name), - "url": escape(misp_url), - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - else: - return {"status": False, - "message": "The key can't be empty", - "name": escape(misp_name), - "url": escape(misp_url), - "apikey": "", - "verifycert": escape(misp_verifycert)} - else: - return {"status": False, - "message": "The url must begin with http:// or https://", - "name": escape(misp_name), - "url": escape(misp_url), - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - else: - return {"status": False, - "message": "The url can't be empty", - "name": escape(misp_name), - "url": "", - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - else: - return {"status": False, - "message": "The MISP instance name can't be empty", - "name":"", - "url": escape(misp_url), - "apikey": escape(misp_key), - "verifycert": escape(misp_verifycert)} - - - @staticmethod - def delete(misp_id): - """ - Delete a MISP instance by its id in the database. - :return: status of the operation in JSON - """ - if db.session.query(exists().where(MISPInst.id == misp_id)).scalar(): - db.session.query(MISPInst).filter_by(id=misp_id).delete() - db.session.commit() - return {"status": True, - "message": "MISP instance deleted"} - else: - return {"status": False, - "message": "MISP instance not found"} - - @staticmethod - def get_all(): - """ - Get all MISP instances from the database - :return: generator of the records. - """ - for mispinstance in db.session.query(MISPInst).all(): - mispinstance = mispinstance.__dict__ - yield {"id": mispinstance["id"], - "name": mispinstance["name"], - "url": mispinstance["url"], - "apikey": mispinstance["apikey"], - "verifycert": mispinstance["verifycert"]} - - @staticmethod - def get_iocs(misp_id, limit, page): - """ - Get all IOCs from the desired MISP instance - :return: generator of the records. - """ - mispinstance = MISPInst.query.get(int(misp_id)) - if mispinstance is not None: - if mispinstance.url != "": - if mispinstance.apikey != "": - try: - # Connects to the desired MISP instance - mispinstance = PyMISP(mispinstance.url, mispinstance.apikey, mispinstance.verifycert) - - # Retreives the attributes (or IOCs) that are supported by Tinycheck - attributes = mispinstance.search('attributes', category='Network activity', limit=limit, page=page, metadata=True) - - - if 'Attribute' in attributes: - iocs = [] - for attribute in attributes['Attribute']: - #print(attribute) - if 'value' in attribute and attribute['value'] != '': - # We have a valid value - ioc_value = attribute['value'] - ioc_type = "unknown" - ioc_tag = "No tag" - ioc_tlp = "white" - isFirstTag = True - - if 'Tag' in attribute: - # We have some tags - #print (attribute['Tag']) - for tag in attribute['Tag']: - tlp = re.search(r"^(?:tlp:)(red|green|amber|white)", tag['name']) - if tlp: - # The current tag is the tlp level - ioc_tlp = tlp.group(1) - #print(ioc_tlp) - elif isFirstTag: - # It is the first retreived tag that is not a tlp - isFirstTag = False - ioc_tag = tag['name'] - else: - # It is another tag and not the first one - ioc_tag += ", " + tag['name'] - - ioc = { "value": ioc_value, - "type": ioc_type, - "tag": ioc_tag, - "tlp": ioc_tlp } - iocs.append(ioc) - return { "status":True, - "results": iocs} - else: - return { "status":False, - "message":"No valid IOCs found."} - except TypeError as error: - print (error) - pass - except: - print("An exception has been raised: ", sys.exc_info()[0]) - pass - else: - {"status": False, - "message": "The api key can't be empty"} - else: - return {"status": False, - "message": "The url can't be empty"} - else: - return {"status": False, - "message": "Unknown MISP instance."} \ No newline at end of file diff --git a/server/backend/app/db/models.py b/server/backend/app/db/models.py index 625091d..c429da8 100644 --- a/server/backend/app/db/models.py +++ b/server/backend/app/db/models.py @@ -1,5 +1,6 @@ from app import db + class Ioc(db.Model): def __init__(self, value, type, tlp, tag, source, added_on): self.value = value @@ -9,6 +10,7 @@ class Ioc(db.Model): self.source = source self.added_on = added_on + class Whitelist(db.Model): def __init__(self, element, type, source, added_on): self.element = element @@ -16,15 +18,16 @@ class Whitelist(db.Model): self.source = source self.added_on = added_on + class MISPInst(db.Model): - def __init__(self, name, url, key, ssl, source, added_on): + def __init__(self, name, url, key, ssl, added_on): self.name = name self.url = url - self.apikey = key + self.authkey = key self.verifycert = ssl - self.source = source self.added_on = added_on + db.mapper(Whitelist, db.Table('whitelist', db.metadata, autoload=True)) db.mapper(Ioc, db.Table('iocs', db.metadata, autoload=True)) -db.mapper(MISPInst, db.Table('mispinstance', db.metadata, autoload=True)) +db.mapper(MISP, db.Table('misp', db.metadata, autoload=True)) diff --git a/server/backend/main.py b/server/backend/main.py index b081caf..eedf327 100644 --- a/server/backend/main.py +++ b/server/backend/main.py @@ -6,6 +6,7 @@ from app.decorators import auth from app.blueprints.ioc import ioc_bp from app.blueprints.whitelist import whitelist_bp from app.blueprints.config import config_bp +from app.blueprints.misp import misp_bp import datetime import secrets import jwt @@ -56,6 +57,7 @@ def page_not_found(e): app.register_blueprint(ioc_bp, url_prefix='/api/ioc') app.register_blueprint(whitelist_bp, url_prefix='/api/whitelist') app.register_blueprint(config_bp, url_prefix='/api/config') +app.register_blueprint(misp_bp, url_prefix='/api/misp') if __name__ == '__main__': ssl_cert = "{}/{}".format(path[0], 'cert.pem') diff --git a/server/backend/watchers.py b/server/backend/watchers.py index fc970b0..60a4c4b 100644 --- a/server/backend/watchers.py +++ b/server/backend/watchers.py @@ -4,6 +4,7 @@ from app.utils import read_config from app.classes.iocs import IOCs from app.classes.whitelist import WhiteList +from app.classes.misp import MISP import requests import json @@ -16,11 +17,6 @@ from multiprocessing import Process in the configuration file. This in order to get automatically new iocs / elements from remote sources without user interaction. - - As of today the default export JSON format from - the backend and unauthenticated HTTP requests - are accepted. The code is little awkward, it'll - be better in a next version ;) """ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -29,7 +25,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) def watch_iocs(): """ Retrieve IOCs from the remote URLs defined in config/watchers. - For each (new ?) IOC, add it to the DB. + For each IOC, add it to the DB. """ # Retrieve the URLs from the configuration @@ -45,8 +41,10 @@ def watch_iocs(): res = requests.get(w["url"], verify=False) if res.status_code == 200: content = json.loads(res.content) - iocs_list = content["iocs"] if "iocs" in content else [] - to_delete = content["to_delete"] if "to_delete" in content else [] + iocs_list = content["iocs"] if "iocs" in content else [ + ] + to_delete = content["to_delete"] if "to_delete" in content else [ + ] else: w["status"] = False except: @@ -93,8 +91,10 @@ def watch_whitelists(): res = requests.get(w["url"], verify=False) if res.status_code == 200: content = json.loads(res.content) - elements = content["elements"] if "elements" in content else [] - to_delete = content["to_delete"] if "to_delete" in content else [] + elements = content["elements"] if "elements" in content else [ + ] + to_delete = content["to_delete"] if "to_delete" in content else [ + ] else: w["status"] = False except: @@ -120,8 +120,25 @@ def watch_whitelists(): break +def watch_misp(): + """ + Retrieve IOCs from misp instances. Each new element is + tested added to the database. + """ + while True: + for misp in MISP.get_instances(): + try: + for ioc in MISP.get_iocs(misp.id): + iocs.add(ioc["type"], ioc["tag"], ioc["tlp"], + ioc["value"], "misp-{}".format(misp["name"])) + except: + continue + + p1 = Process(target=watch_iocs) p2 = Process(target=watch_whitelists) +p3 = Process(target=watch_misp) p1.start() p2.start() +p3.start()