From 08a4f26de4962362a71f21d73a74509d058130f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Aime?= Date: Mon, 14 Jun 2021 17:06:45 +0200 Subject: [PATCH] First OpenCTI implementation dev --- assets/scheme.sql | 11 ++ server/backend/app/classes/octi.py | 164 +++++++++++++++++++++++++++++ server/backend/app/db/models.py | 11 ++ server/backend/watchers.py | 42 +++++++- 4 files changed, 223 insertions(+), 5 deletions(-) create mode 100644 server/backend/app/classes/octi.py diff --git a/assets/scheme.sql b/assets/scheme.sql index a689cb6..e4e8812 100644 --- a/assets/scheme.sql +++ b/assets/scheme.sql @@ -28,3 +28,14 @@ CREATE TABLE "misp" ( "last_sync" NUMERIC NOT NULL DEFAULT 0, PRIMARY KEY("id" AUTOINCREMENT) ); + +CREATE TABLE "octi" ( + "id" INTEGER UNIQUE, + "name" TEXT, + "url" TEXT NOT NULL, + "apikey" TEXT NOT NULL, + "verifycert" INTEGER NOT NULL DEFAULT 0, + "added_on" NUMERIC NOT NULL, + "last_sync" NUMERIC NOT NULL DEFAULT 0, + PRIMARY KEY("id" AUTOINCREMENT) +); diff --git a/server/backend/app/classes/octi.py b/server/backend/app/classes/octi.py new file mode 100644 index 0000000..2db4aad --- /dev/null +++ b/server/backend/app/classes/octi.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from app import db +from app.db.models import OCTIInst +from app.definitions import definitions as defs + +from sqlalchemy.sql import exists +from urllib.parse import unquote +from flask import escape +from pycti import OpenCTIApiClient, Infrastructure +import re +import time +import sys + + +class OCTI(object): + def __init__(self): + return None + + def add_instance(self, instance): + """ + Parse and add a OpenCTI instance to the database. + :return: status of the operation in JSON + """ + + url = instance["url"] + name = instance["name"] + apikey = instance["key"] + verify = instance["ssl"] + last_sync = int(time.time()-31536000) # One year + + sameinstances = db.session.query(OCTIInst).filter( + OCTIInst.url == url, OCTIInst.apikey == apikey) + if sameinstances.count(): + return {"status": False, + "message": "This OpenCTI instance already exists"} + if name: + if self.test_instance(url, apikey, verify): + added_on = int(time.time()) + db.session.add(OCTIInst(name, escape( + url), apikey, verify, added_on, last_sync)) + db.session.commit() + return {"status": True, + "message": "OpenCTI instance added"} + else: + return {"status": False, + "message": "Please verify the connection to the OpenCTI instance"} + else: + return {"status": False, + "message": "Please provide a name for your instance"} + + @staticmethod + def delete_instance(opencti_id): + """ + Delete a OpenCTI instance by its id in the database. + :return: status of the operation in JSON + """ + if db.session.query(exists().where(OCTIInst.id == misp_id)).scalar(): + db.session.query(OCTIInst).filter_by(id=misp_id).delete() + db.session.commit() + return {"status": True, + "message": "OpenCTI instance deleted"} + else: + return {"status": False, + "message": "OpenCTI instance not found"} + + def get_instances(self): + """ + Get OpenCTI instances from the database + :return: generator of the records. + """ + for opencti in db.session.query(OCTIInst).all(): + opencti = opencti.__dict__ + yield {"id": opencti["id"], + "name": opencti["name"], + "url": opencti["url"], + "apikey": opencti["apikey"], + "verifycert": True if opencti["verifycert"] else False, + "connected": self.test_instance(opencti["url"], opencti["apikey"], opencti["verifycert"]), + "lastsync": opencti["last_sync"]} + + @staticmethod + def test_instance(url, apikey, verify): + """ + Test the connection of the OpenCTI instance. + :return: generator of the records. + """ + try: + OpenCTIApiClient(url, token=apikey, ssl_verify=verify) + return True + except: + return False + + @staticmethod + def update_sync(opencti_id): + """ + Update the last synchronization date by the actual date. + :return: bool, True if updated. + """ + try: + misp = OCTIInst.query.get(int(opencti_id)) + misp.last_sync = int(time.time()) + db.session.commit() + return True + except: + return False + + @staticmethod + def get_iocs(opencti_id): + """ + Get all IOCs from specific OpenCTI instance + :return: generator containing the IOCs. + """ + opencti = OCTIInst.query.get(int(opencti_id)) + if opencti is not None: + if opencti.url and opencti.apikey: + try: + # Connect to OpenCTI instance and get network activity attributes. + i = OpenCTIApiClient( + opencti.url, opencti.apikey, opencti.verifycert) + r = Infrastructure(i).list(getAll=True) + except: + print( + "Unable to connect to the OpenCTI instance ({}/{}).".format(opencti.url, opencti.apikey)) + return [] + """ + for attr in r["Attribute"]: + if attr["type"] in ["ip-dst", "domain", "snort", "x509-fingerprint-sha1"]: + + ioc = {"value": attr["value"], + "type": None, + "tag": "suspect", + "tlp": "white"} + + # Deduce the IOC type. + if re.match(defs["iocs_types"][0]["regex"], attr["value"]): + ioc["type"] = "ip4addr" + elif re.match(defs["iocs_types"][1]["regex"], attr["value"]): + ioc["type"] = "ip6addr" + elif re.match(defs["iocs_types"][2]["regex"], attr["value"]): + ioc["type"] = "cidr" + elif re.match(defs["iocs_types"][3]["regex"], attr["value"]): + ioc["type"] = "domain" + elif re.match(defs["iocs_types"][4]["regex"], attr["value"]): + ioc["type"] = "sha1cert" + elif "alert " in attr["value"][0:6]: + ioc["type"] = "snort" + else: + continue + + if "Tag" in attr: + for tag in attr["Tag"]: + # Add a TLP to the IOC if defined in tags. + tlp = re.search( + r"^(?:tlp:)(red|green|amber|white)", tag['name'].lower()) + if tlp: + ioc["tlp"] = tlp.group(1) + + # Add possible tag (need to match TinyCheck tags) + if tag["name"].lower() in [t["tag"] for t in defs["iocs_tags"]]: + ioc["tag"] = tag["name"].lower() + yield ioc + """ diff --git a/server/backend/app/db/models.py b/server/backend/app/db/models.py index e965caa..ecd0665 100644 --- a/server/backend/app/db/models.py +++ b/server/backend/app/db/models.py @@ -29,6 +29,17 @@ class MISPInst(db.Model): self.last_sync = last_sync +class OCTIInst(db.Model): + def __init__(self, name, url, key, ssl, added_on, last_sync): + self.name = name + self.url = url + self.apikey = key + self.verifycert = ssl + self.added_on = added_on + self.last_sync = last_sync + + db.mapper(Whitelist, db.Table('whitelist', db.metadata, autoload=True)) db.mapper(Ioc, db.Table('iocs', db.metadata, autoload=True)) db.mapper(MISPInst, db.Table('misp', db.metadata, autoload=True)) +db.mapper(OCTIInst, db.Table('octi', db.metadata, autoload=True)) diff --git a/server/backend/watchers.py b/server/backend/watchers.py index 7299eb8..b884ca9 100644 --- a/server/backend/watchers.py +++ b/server/backend/watchers.py @@ -5,6 +5,7 @@ from app.utils import read_config from app.classes.iocs import IOCs from app.classes.whitelist import WhiteList from app.classes.misp import MISP +from app.classes.octi import OCTI import requests import json @@ -41,8 +42,10 @@ def watch_iocs(): res = requests.get(w["url"], verify=False) if res.status_code == 200: content = json.loads(res.content) - iocs_list = content["iocs"] if "iocs" in content else [] - to_delete = content["to_delete"] if "to_delete" in content else [] + iocs_list = content["iocs"] if "iocs" in content else [ + ] + to_delete = content["to_delete"] if "to_delete" in content else [ + ] else: w["status"] = False except: @@ -89,8 +92,10 @@ def watch_whitelists(): res = requests.get(w["url"], verify=False) if res.status_code == 200: content = json.loads(res.content) - elements = content["elements"] if "elements" in content else [] - to_delete = content["to_delete"] if "to_delete" in content else [] + elements = content["elements"] if "elements" in content else [ + ] + to_delete = content["to_delete"] if "to_delete" in content else [ + ] else: w["status"] = False except: @@ -135,13 +140,40 @@ def watch_misp(): ioc["value"], "misp-{}".format(ist["id"])) misp.update_sync(ist["id"]) instances.pop(i) - if instances: time.sleep(60) + if instances: + time.sleep(60) + + +def watch_opencti(): + """ + Retrieve IOCs from OpenCTI instances. Each new element is + tested and then added to the database. + """ + iocs, octi = IOCs(), OCTI() + instances = [i for i in octi.get_instances()] + + while instances: + for i, ist in enumerate(instances): + status = octi.test_instance(ist["url"], + ist["apikey"], + ist["verifycert"]) + if status: + print("Testing...") + # for ioc in octi.get_iocs(ist["id"]): + # iocs.add(ioc["type"], ioc["tag"], ioc["tlp"], + # ioc["value"], "octi-{}".format(ist["id"])) + # octi.update_sync(ist["id"]) + instances.pop(i) + if instances: + time.sleep(60) p1 = Process(target=watch_iocs) p2 = Process(target=watch_whitelists) p3 = Process(target=watch_misp) +p4 = Process(target=watch_octi) p1.start() p2.start() p3.start() +p4.start()