Working on the MISP integration, still WIP (untested)

server/backend/app/blueprints/misp.py

@@ -3,28 +3,30 @@
 
 from flask import Blueprint, jsonify, Response, request
 from app.decorators import require_header_token, require_get_token
-from app.classes.mispobj import MISPObj
+from app.classes.misp import MISP
 
 import json
 
 misp_bp = Blueprint("misp", __name__)
-misp = MISPObj()
+misp = MISP()
 
 
 @misp_bp.route('/add', methods=['POST'])
 @require_header_token
-def add ():
+def add():
     """
         Parse and add a MISP instance to the database.
         :return: status of the operation in JSON
     """
     data = json.loads(request.data)
     instance = data["data"]["instance"]
-    
+
     source = "backend"
-    res = MISPObj.add(instance["name"], instance["url"], instance["key"], instance["ssl"], source)
+    res = MISP.add(instance["name"], instance["url"],
+                   instance["key"], instance["ssl"], source)
     return jsonify(res)
 
+
 @misp_bp.route('/delete/<misp_id>', methods=['GET'])
 @require_header_token
 def delete(misp_id):
@@ -32,9 +34,10 @@ def delete(misp_id):
         Delete a MISP instance by its id to the database.
         :return: status of the operation in JSON
     """
-    res = MISPObj.delete(misp_id)
+    res = MISP.delete(misp_id)
     return jsonify(res)
 
+
 @misp_bp.route('/get_all', methods=['GET'])
 @require_header_token
 def get_all():
@@ -42,12 +45,12 @@ def get_all():
         Retreive a list of all MISP instances.
         :return: list of MISP instances in JSON.
     """
-    res = MISPObj.get_all()
+    res = MISP.get_all()
     return jsonify({"results": [i for i in res]})
 
 
 @misp_bp.route('/get_iocs', methods=['POST'])
-#@require_header_token
+# @require_header_token
 def get_iocs():
     """
         Retreive a list of all MISP instances.
@@ -57,20 +60,24 @@ def get_iocs():
     data = json.loads(request.data)
     data = data["data"]
 
-    res = MISPObj.get_iocs(data["misp_id"], data["limit"], data["page"])
+    res = MISP.get_iocs(data["misp_id"],
-    print(res)
+                        data["limit"],
+                        data["page"])
     return jsonify(res)
 
 
 @misp_bp.route('/edit', methods=['POST'])
 @require_header_token
-def edit ():
+def edit():
     """
         Parse and edit the desired MISP instance.
         :return: status of the operation in JSON
     """
     data = json.loads(request.data)
     instance = data["data"]["instance"]
-    print(instance)
+    res = MISP.edit(instance["id"],
-    res = MISPObj.edit(instance["id"], instance["name"], instance["url"], instance["apikey"], instance["verifycert"])
+                    instance["name"],
-    return jsonify(res)
+                    instance["url"],
+                    instance["apikey"],
+                    instance["verifycert"])
+    return jsonify(res)

server/backend/app/classes/misp.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from app import db
+from app.db.models import MISPInst
+from sqlalchemy.sql import exists
+from app.definitions import definitions as defs
+from urllib.parse import unquote
+from flask import escape
+from pymisp import PyMISP
+import re
+import time
+import sys
+
+
+class MISP(object):
+    def __init__(self):
+        return None
+
+    @staticmethod
+    def add(misp_name, misp_url, misp_key, misp_verifycert):
+        """
+            Parse and add a MISP instance to the database.
+            :return: status of the operation in JSON
+        """
+
+        sameinstances = db.session.query(MISPInst).filter(
+            MISPInst.url == misp_url, MISPInst.apikey == misp_key)
+        if sameinstances.count():
+            return {"status": False,
+                    "message": "This MISP instance already exists"}
+        elif misp_name != "":
+            if misp_url != "":
+                if re.match(r"^(?:(?:http|https)://)", misp_url):
+                    if misp_key != "":
+                        added_on = int(time.time())
+                        db.session.add(MISPInst(misp_name, escape(
+                            misp_url), misp_key, misp_verifycert, added_on))
+                        db.session.commit()
+                        return {"status": True,
+                                "message": "MISP instance added",
+                                "name": escape(misp_name),
+                                "url": escape(misp_url),
+                                "apikey": escape(misp_key),
+                                "verifycert": escape(misp_verifycert)}
+                    else:
+                        return {"status": False,
+                                "message": "The key can't be empty"}
+                else:
+                    return {"status": False,
+                            "message": "The url must begin with http:// or https://"}
+            else:
+                return {"status": False,
+                        "message": "The url can't be empty"}
+        else:
+            return {"status": False,
+                    "message": "The MISP instance name can't be empty"}
+
+    @staticmethod
+    def edit(misp_id, misp_name, misp_url, misp_key, misp_verifycert):
+        """
+            Parse and edit the desired MISP instance.
+            :return: status of the operation in JSON
+        """
+        misp = MISPInst.query.get(int(misp_id))
+        otherinstances = db.session.query(MISPInst).filter(MISPInst.id != int(
+            misp_id), MISPInst.url == misp_url, MISPInst.apikey == misp_key)
+        if misp is None:
+            return {"status": False,
+                    "message": "Can't find the MISP instance"}
+        if otherinstances.count() > 0:
+            return {"status": False,
+                    "message": "This MISP instance already exists"}
+        elif misp_name != "":
+            if misp_url != "":
+                if re.match(r"^(?:(?:http|https)://)", misp_url):
+                    if misp_key != "":
+                        misp.name = misp_name
+                        misp.url = misp_url
+                        misp.apikey = misp_key
+                        misp.verifycert = misp_verifycert
+                        db.session.commit()
+                        return {"status": True,
+                                "message": "MISP instance edited"}
+                    else:
+                        return {"status": False,
+                                "message": "The key can't be empty"}
+                else:
+                    return {"status": False,
+                            "message": "The url must begin with http:// or https://"}
+            else:
+                return {"status": False,
+                        "message": "The url can't be empty"}
+        else:
+            return {"status": False,
+                    "message": "The MISP instance name can't be empty"}
+
+    @staticmethod
+    def delete_instance(misp_id):
+        """
+            Delete a MISP instance by its id in the database.
+            :return: status of the operation in JSON
+        """
+        if db.session.query(exists().where(MISPInst.id == misp_id)).scalar():
+            db.session.query(MISPInst).filter_by(id=misp_id).delete()
+            db.session.commit()
+            return {"status": True,
+                    "message": "MISP instance deleted"}
+        else:
+            return {"status": False,
+                    "message": "MISP instance not found"}
+
+    @staticmethod
+    def get_instances():
+        """
+            Get MISP instances from the database
+            :return: generator of the records.
+        """
+        for misp in db.session.query(MISPInst).all():
+            misp = misp.__dict__
+            yield {"id": misp["id"],
+                   "name": misp["name"],
+                   "url": misp["url"],
+                   "apikey": misp["apikey"],
+                   "verifycert": misp["verifycert"]}
+
+    @staticmethod
+    def get_iocs(misp_id):
+        """
+            Get all IOCs from specific MISP instance
+            /!\ Todo: NEED TO ADD LAST SYNCHRO DATE + page etc. stuff.
+            :return: generator containing the IOCs.
+        """
+        misp = MISPInst.query.get(int(misp_id))
+        if misp is not None:
+            if misp.url and misp.apikey:
+                try:
+                    # Connect to MISP instance and get network activity attributes.
+                    m = PyMISP(misp.url, misp.apikey, misp.verifycert)
+                    r = m.search("attributes", category="Network activity")
+
+                    for attr in r["Attribute"]:
+                        if attr["type"] in ["ip_dst", "domain", "snort", "x509-fingerprint-sha1"]:
+
+                            ioc = {"value": attr["value"],
+                                   "type": None,
+                                   "tag": "suspect",
+                                   "tlp": "white"}
+
+                            # Deduce the IOC type.
+                            if re.match(defs["iocs_types"][0]["regex"], attr["value"]):
+                                ioc["type"] = "ipv4addr"
+                            elif re.match(defs["iocs_types"][1]["regex"], attr["value"]):
+                                ioc["type"] = "ipv6addr"
+                            elif re.match(defs["iocs_types"][3]["regex"], attr["value"]):
+                                ioc["type"] = "domain"
+                            elif re.match(defs["iocs_types"][4]["regex"], attr["value"]):
+                                ioc["type"] = "sha1cert"
+                            elif "alert " in attr["value"][0:5]:
+                                ioc["type"] = "snort"
+
+                            if "Tag" in attr:
+                                for tag in attribute['Tag']:
+                                    # Add the TLP of the IOC.
+                                    tlp = re.search(r"^(?:tlp:)(red|green|amber|white)", tag['name'])
+                                    if tlp: ioc["tlp"] = tlp.group(1)
+
+                                    # Add possible tag.
+                                    if lower(tag["name"]) in [t["tag"] for t in defs["iocs_tags"]]:
+                                        ioc["tag"] = lower(tag["name"])
+                            yield ioc
+                except:
+                    return {"status": False,
+                            "message": "An exception has been raised: ", sys.exc_info()[0])}
+                    pass
+            else:
+                return {"status": False,
+                        "message": "The URL or API key is empty."}
+        else:
+            return {"status": False,
+                    "message": "Unknown MISP instance."}

server/backend/app/classes/mispobj.py

@@ -1,240 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from app import db
-from app.db.models import MISPInst
-from sqlalchemy.sql import exists
-from app.definitions import definitions
-from urllib.parse import unquote
-from flask import escape
-from pymisp import PyMISP
-import re
-import time
-import sys
-
-
-class MISPObj(object):
-    def __init__(self):
-        return None
-
-    @staticmethod
-    def add(misp_name, misp_url, misp_key, misp_verifycert, source):
-        """
-            Parse and add a MISP"instance to the database.
-            :return: status of the operation in JSON
-        """
-        
-        sameinstances = db.session.query(MISPInst).filter(MISPInst.url == misp_url, MISPInst.apikey == misp_key)
-        if sameinstances.count() > 0:
-            return {"status": False,
-                    "message": "This MISP instance already exists",
-                    "name": escape(misp_name),
-                    "url": escape(misp_url),
-                    "apikey": escape(misp_key),
-                    "verifycert": escape(misp_verifycert)}
-        elif misp_name != "":          
-            if misp_url != "":               
-                if re.match(r"^(?:(?:http|https)://)", misp_url):
-                    if misp_key != "":
-                        added_on = int(time.time())
-                        db.session.add(MISPInst(misp_name, escape(misp_url), misp_key, misp_verifycert, source, added_on))
-                        db.session.commit()
-                        return {"status": True,
-                            "message": "MISP instance added",
-                            "name": escape(misp_name),
-                            "url": escape(misp_url),
-                            "apikey": escape(misp_key),
-                            "verifycert": escape(misp_verifycert)}
-                    else:
-                        return {"status": False,
-                        "message": "The key can't be empty",
-                        "name": escape(misp_name),
-                        "url": escape(misp_url),
-                        "apikey": "",
-                        "verifycert": escape(misp_verifycert)}
-                else:
-                    return {"status": False,
-                        "message": "The url must begin with http:// or https://",
-                        "name": escape(misp_name),
-                        "url": escape(misp_url),
-                        "apikey": escape(misp_key),
-                        "verifycert": escape(misp_verifycert)}
-            else:
-                return {"status": False,
-                        "message": "The url can't be empty",
-                        "name": escape(misp_name),
-                        "url": "",
-                        "apikey": escape(misp_key),
-                        "verifycert": escape(misp_verifycert)}
-        else:
-            return {"status": False,
-                    "message": "The MISP instance name can't be empty",
-                    "name":"",
-                    "url": escape(misp_url),
-                    "apikey": escape(misp_key),
-                    "verifycert": escape(misp_verifycert)}
-
-    @staticmethod
-    def edit(misp_id, misp_name, misp_url, misp_key, misp_verifycert):
-        """
-            Parse and edit the desired MISP instance.
-            :return: status of the operation in JSON
-        """
-        mispinstance = MISPInst.query.get(int(misp_id))
-        otherinstances = db.session.query(MISPInst).filter(MISPInst.id != int(misp_id), MISPInst.url == misp_url, MISPInst.apikey == misp_key)
-        if mispinstance is None:                                                                                                                                            
-            return {"status": False,
-                    "message": "Can't find the MISP instance"}
-        if otherinstances.count() > 0:
-            return {"status": False,
-                    "message": "This MISP instance already exists",
-                    "name": escape(misp_name),
-                    "url": escape(misp_url),
-                    "apikey": escape(misp_key),
-                    "verifycert": escape(misp_verifycert)}
-        elif misp_name != "":          
-            if misp_url != "":               
-                if re.match(r"^(?:(?:http|https)://)", misp_url):
-                    if misp_key != "":
-                        mispinstance.name = misp_name
-                        mispinstance.url = misp_url
-                        mispinstance.apikey = misp_key
-                        mispinstance.verifycert = misp_verifycert
-                        db.session.commit()
-                        return {"status": True,
-                            "message": "MISP instance edited",
-                            "name": escape(misp_name),
-                            "url": escape(misp_url),
-                            "apikey": escape(misp_key),
-                            "verifycert": escape(misp_verifycert)}
-                    else:
-                        return {"status": False,
-                        "message": "The key can't be empty",
-                        "name": escape(misp_name),
-                        "url": escape(misp_url),
-                        "apikey": "",
-                        "verifycert": escape(misp_verifycert)}
-                else:
-                    return {"status": False,
-                        "message": "The url must begin with http:// or https://",
-                        "name": escape(misp_name),
-                        "url": escape(misp_url),
-                        "apikey": escape(misp_key),
-                        "verifycert": escape(misp_verifycert)}
-            else:
-                return {"status": False,
-                        "message": "The url can't be empty",
-                        "name": escape(misp_name),
-                        "url": "",
-                        "apikey": escape(misp_key),
-                        "verifycert": escape(misp_verifycert)}
-        else:
-            return {"status": False,
-                    "message": "The MISP instance name can't be empty",
-                    "name":"",
-                    "url": escape(misp_url),
-                    "apikey": escape(misp_key),
-                    "verifycert": escape(misp_verifycert)}
-
-
-    @staticmethod
-    def delete(misp_id):
-        """
-            Delete a MISP instance by its id in the database.
-            :return: status of the operation in JSON
-        """
-        if db.session.query(exists().where(MISPInst.id == misp_id)).scalar():
-            db.session.query(MISPInst).filter_by(id=misp_id).delete()
-            db.session.commit()
-            return {"status": True,
-                    "message": "MISP instance deleted"}
-        else:
-            return {"status": False,
-                    "message": "MISP instance not found"}
-
-    @staticmethod
-    def get_all():
-        """
-            Get all MISP instances from the database
-            :return: generator of the records.
-        """
-        for mispinstance in db.session.query(MISPInst).all():
-            mispinstance = mispinstance.__dict__
-            yield {"id": mispinstance["id"],
-                   "name": mispinstance["name"],
-                   "url": mispinstance["url"],
-                   "apikey": mispinstance["apikey"],
-                   "verifycert": mispinstance["verifycert"]}
-
-    @staticmethod
-    def get_iocs(misp_id, limit, page):
-        """
-            Get all IOCs from the desired MISP instance
-            :return: generator of the records.
-        """
-        mispinstance = MISPInst.query.get(int(misp_id))
-        if mispinstance is not None:
-            if mispinstance.url != "":
-                if mispinstance.apikey != "":
-                    try:
-                        # Connects to the desired MISP instance
-                        mispinstance = PyMISP(mispinstance.url, mispinstance.apikey, mispinstance.verifycert)
-                        
-                        # Retreives the attributes (or IOCs) that are supported by Tinycheck
-                        attributes = mispinstance.search('attributes', category='Network activity', limit=limit, page=page, metadata=True)
-                        
-
-                        if 'Attribute' in attributes:
-                            iocs = []
-                            for attribute in attributes['Attribute']:
-                                #print(attribute)
-                                if 'value' in attribute and attribute['value'] != '':
-                                    # We have a valid value
-                                    ioc_value = attribute['value']
-                                    ioc_type = "unknown"
-                                    ioc_tag = "No tag"
-                                    ioc_tlp = "white"
-                                    isFirstTag = True
-                                    
-                                    if 'Tag' in attribute:
-                                        # We have some tags
-                                        #print (attribute['Tag'])
-                                        for tag in attribute['Tag']:
-                                            tlp = re.search(r"^(?:tlp:)(red|green|amber|white)", tag['name'])
-                                            if tlp:
-                                                # The current tag is the tlp level
-                                                ioc_tlp = tlp.group(1)
-                                                #print(ioc_tlp)
-                                            elif isFirstTag:
-                                                # It is the first retreived tag that is not a tlp 
-                                                isFirstTag = False
-                                                ioc_tag = tag['name']
-                                            else:
-                                                # It is another tag and not the first one
-                                                ioc_tag += ", " + tag['name']
-                                                
-                                    ioc = { "value": ioc_value,
-                                            "type": ioc_type,
-                                            "tag": ioc_tag,
-                                            "tlp": ioc_tlp }
-                                    iocs.append(ioc)
-                            return { "status":True,
-                                     "results": iocs}
-                        else:
-                            return { "status":False,
-                                     "message":"No valid IOCs found."}
-                    except TypeError as error:
-                        print (error)
-                        pass
-                    except:
-                        print("An exception has been raised: ", sys.exc_info()[0])
-                        pass
-                else:
-                    {"status": False,
-                     "message": "The api key can't be empty"}       
-            else:
-                return {"status": False,
-                        "message": "The url can't be empty"}
-        else:
-            return {"status": False,
-                    "message": "Unknown MISP instance."}

server/backend/app/db/models.py

@@ -1,5 +1,6 @@
 from app import db
 
+
 class Ioc(db.Model):
     def __init__(self, value, type, tlp, tag, source, added_on):
         self.value = value
@@ -9,6 +10,7 @@ class Ioc(db.Model):
         self.source = source
         self.added_on = added_on
 
+
 class Whitelist(db.Model):
     def __init__(self, element, type, source, added_on):
         self.element = element
@@ -16,15 +18,16 @@ class Whitelist(db.Model):
         self.source = source
         self.added_on = added_on
 
+
 class MISPInst(db.Model):
-    def __init__(self, name, url, key, ssl, source, added_on):
+    def __init__(self, name, url, key, ssl, added_on):
         self.name = name
         self.url = url
-        self.apikey = key
+        self.authkey = key
         self.verifycert = ssl
-        self.source = source
         self.added_on = added_on
 
+
 db.mapper(Whitelist, db.Table('whitelist', db.metadata, autoload=True))
 db.mapper(Ioc, db.Table('iocs', db.metadata, autoload=True))
-db.mapper(MISPInst, db.Table('mispinstance', db.metadata, autoload=True))
+db.mapper(MISP, db.Table('misp', db.metadata, autoload=True))

server/backend/main.py

@@ -6,6 +6,7 @@ from app.decorators import auth
 from app.blueprints.ioc import ioc_bp
 from app.blueprints.whitelist import whitelist_bp
 from app.blueprints.config import config_bp
+from app.blueprints.misp import misp_bp
 import datetime
 import secrets
 import jwt
@@ -56,6 +57,7 @@ def page_not_found(e):
 app.register_blueprint(ioc_bp, url_prefix='/api/ioc')
 app.register_blueprint(whitelist_bp, url_prefix='/api/whitelist')
 app.register_blueprint(config_bp, url_prefix='/api/config')
+app.register_blueprint(misp_bp, url_prefix='/api/misp')
 
 if __name__ == '__main__':
     ssl_cert = "{}/{}".format(path[0], 'cert.pem')

server/backend/watchers.py

@@ -4,6 +4,7 @@
 from app.utils import read_config
 from app.classes.iocs import IOCs
 from app.classes.whitelist import WhiteList
+from app.classes.misp import MISP
 
 import requests
 import json
@@ -16,11 +17,6 @@ from multiprocessing import Process
     in the configuration file. This in order to get
     automatically new iocs / elements from remote
     sources without user interaction.
-
-    As of today the default export JSON format from
-    the backend and unauthenticated HTTP requests
-    are accepted. The code is little awkward, it'll
-    be better in a next version ;)
 """
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -29,7 +25,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 def watch_iocs():
     """
         Retrieve IOCs from the remote URLs defined in config/watchers.
-        For each (new ?) IOC, add it to the DB.
+        For each IOC, add it to the DB.
     """
 
     # Retrieve the URLs from the configuration
@@ -45,8 +41,10 @@ def watch_iocs():
                     res = requests.get(w["url"], verify=False)
                     if res.status_code == 200:
                         content = json.loads(res.content)
-                        iocs_list = content["iocs"] if "iocs" in content else []
+                        iocs_list = content["iocs"] if "iocs" in content else [
-                        to_delete = content["to_delete"] if "to_delete" in content else []
+                        ]
+                        to_delete = content["to_delete"] if "to_delete" in content else [
+                        ]
                     else:
                         w["status"] = False
                 except:
@@ -93,8 +91,10 @@ def watch_whitelists():
                     res = requests.get(w["url"], verify=False)
                     if res.status_code == 200:
                         content = json.loads(res.content)
-                        elements = content["elements"] if "elements" in content else []
+                        elements = content["elements"] if "elements" in content else [
-                        to_delete = content["to_delete"] if "to_delete" in content else []
+                        ]
+                        to_delete = content["to_delete"] if "to_delete" in content else [
+                        ]
                     else:
                         w["status"] = False
                 except:
@@ -120,8 +120,25 @@ def watch_whitelists():
             break
 
 
+def watch_misp():
+    """
+        Retrieve IOCs from misp instances. Each new element is
+        tested added to the database.
+    """
+    while True:
+        for misp in MISP.get_instances():
+            try:
+                for ioc in MISP.get_iocs(misp.id):
+                    iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
+                             ioc["value"], "misp-{}".format(misp["name"]))
+            except:
+                continue
+
+
 p1 = Process(target=watch_iocs)
 p2 = Process(target=watch_whitelists)
+p3 = Process(target=watch_misp)
 
 p1.start()
 p2.start()
+p3.start()