Update on iocs-misp

app/backend/src/App.vue

@@ -42,6 +42,9 @@
                 <li class="menu-item">
                   <span @click="$router.push('/iocs/misp')">MISP Instances</span>
                 </li>
+                <li class="menu-item">
+                  <span @click="$router.push('/iocs/opencti')">OpenCTI Instances</span>
+                </li>
               </ul>
             </div>
           </div>

app/backend/src/router/index.js

@@ -40,6 +40,12 @@ const routes = [
     component: () => import('../views/iocs-misp.vue'),
     props: true
   },
+  {
+    path: '/iocs/opencti',
+    name: 'iocs-opencti',
+    component: () => import('../views/iocs-opencti.vue'),
+    props: true
+  },
   {
     path: '/iocs/search',
     name: 'iocs-search',

app/backend/src/views/iocs-misp.vue

@@ -18,16 +18,16 @@
                     <input class="form-input" type="text" ref="misp_url" placeholder="https://misp.cyberacme.com" v-model="mispinst.url" required>
                     <label class="misp-label">Authentication key</label><span></span>
                     <input class="form-input" type="text" ref="misp_key" placeholder="OqHSMyAuth3ntic4t10nK3y0MyAuth3ntic4t10nK3y3iiH" v-model="mispinst.key" required>
-                    <label class="misp-label">Verify certificate? </label><span></span>
+                    <label class="misp-label" v-if="mispinst.url.startsWith('https://')">Verify certificate? </label><span  v-if="mispinst.url.startsWith('https://')"></span>
-                    <div style="flex:50%"><label class="form-switch">
+                    <div style="flex:50%" v-if="mispinst.url.startsWith('https://')"><label class="form-switch">
-                    <input type="checkbox" @change="switch_config('frontend', 'kiosk_mode')" v-model="mispinst.ssl">
+                    <input type="checkbox" v-model="mispinst.ssl">
                     <i class="form-icon"></i>
                     </label></div>
                 </div>
                 <button class="btn-primary btn col-12" v-on:click="add_instance()">Add MISP instance</button>
                 <div class="form-group" v-if="added">
                     <div class="toast toast-success">
-                        ✓ MISP instance added successfully.
+                        ✓ MISP instance added successfully. Redirecting to instances in 2 seconds. 
                     </div>
                 </div>
                 <div class="form-group" v-if="error">
@@ -108,6 +108,10 @@ export default {
                 axios.post(`/api/misp/add`, { data: { instance: this.mispinst } }, { headers: {'X-Token': this.jwt} }).then(response => {
                     if(response.data.status){
                         this.added = true;
+                        setTimeout(function (){ 
+                            this.switch_tab('instances')
+                            this.mispinst = { name:'', url:'',key:'', ssl:false } 
+                        }.bind(this), 2000);
                     } else {
                         this.error = response.data.message;
                     }

app/backend/src/views/iocs-opencti.vue

@@ -0,0 +1,170 @@
+<template>
+    <div class="backend-content" id="content">
+        <div class="column col-8 col-xs-12">
+            <h3 class="s-title">Manage OpenCTI instances</h3>
+            <ul class="tab tab-block">
+                <li class="tab-item">
+                    <a href="#" v-on:click="switch_tab('addopencti')" v-bind:class="{ active: tabs.addopencti }">Add instance</a>
+                </li>
+                <li class="tab-item">
+                    <a href="#" v-on:click="switch_tab('instances')" v-bind:class="{ active: tabs.instances }">Existing instances</a>
+                </li>
+            </ul>
+            <div v-if="tabs.addopencti">
+                <div class="misp-form">
+                    <label class="misp-label">Instance name</label><span></span>
+                    <input class="form-input" type="text" placeholder="CYBERACME OpenCTI" v-model="openctiinst.name" required>
+                    <label class="misp-label">Instance URL</label><span></span>
+                    <input class="form-input" type="text" placeholder="https://opencti.cyberacme.com" v-model="openctiinst.url" required>
+                    <label class="misp-label">Authentication key</label><span></span>
+                    <input class="form-input" type="text" placeholder="83114ab2-3570-493b-8caa-14ef1bcf8e9a" v-model="openctiinst.key" required>
+                    <label class="misp-label">Verify certificate? </label><span></span>
+                    <div style="flex:50%"><label class="form-switch">
+                    <input type="checkbox" v-model="openctiinst.ssl">
+                    <i class="form-icon"></i>
+                    </label></div>
+                </div>
+                <button class="btn-primary btn col-12" v-on:click="add_instance()">Add OpenCTI instance</button>
+                <div class="form-group" v-if="added">
+                    <div class="toast toast-success">
+                        ✓ OpenCTI instance added successfully.
+                    </div>
+                </div>
+                <div class="form-group" v-if="error">
+                    <div class="toast toast-error">
+                        ✗ OpenCTI instance not added. {{error}}
+                    </div>
+                </div>
+            </div>
+            <div class="form-group" v-if="tabs.instances">
+                <div v-if="instances.length">
+                    <table class="table table-striped table-hover">
+                        <thead>
+                            <tr>
+                                <th>Name</th>
+                                <th>Server</th>
+                                <th>Authkey</th>
+                                <th>Status</th>
+                                <th>Action</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            <tr v-for="i in instances" v-bind:key="i.id">
+                                <td>{{ i.name }}</td>
+                                <td>{{ i.url.replace('https://', '') .replace('http://', '') }}</td>
+                                <td>{{ i.apikey.slice(0,5) }} [...] {{ i.apikey.slice(35,40) }}</td>
+                                <td>
+                                    <span v-if="i.connected" class="misp-online tooltip" :data-tooltip="i.lastsync">✓ ONLINE</span>
+                                    <span v-else class="misp-offline tooltip" :data-tooltip="i.lastsync">⚠ OFFLINE</span>
+                                </td>
+                                <td><button class="btn btn-sm" v-on:click="delete_instance(i)">Delete</button></td>
+                            </tr>
+                        </tbody>
+                    </table>
+                </div>
+                <div v-else>
+                    <div class="empty">
+                        <div v-if="loading">
+                            <p class="empty-title h5">
+                                <span class="loading loading-lg"></span>
+                            </p>
+                            <p class="empty-subtitle">Testing and loading your OpenCTI instances.</p>
+                        </div>
+                        <div v-else>
+                            <p class="empty-title h5">No OpenCTI instance found.</p>
+                            <p class="empty-subtitle">Do not hesitate to add a OpenCTI instance.</p>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</template>
+<script>
+
+import axios from 'axios'
+
+export default {
+    name: 'manageopencti',   
+    data() {
+        return { 
+            error:false,
+            loading:false,
+            added:false,
+            openctiinst:{ name:'', url:'',key:'', ssl:false },
+            instances:[],
+            tabs: { "addopencti" : true, "instances" : false },
+            jwt:""
+        }
+    },
+    props: { },
+    methods: {
+        add_instance: function()
+        {  
+            this.added = false;
+            this.error = false;
+            if (this.openctiinst.name && this.openctiinst.url && this.openctiinst.key)
+            {
+                axios.post(`/api/opencti/add`, { data: { instance: this.openctiinst } }, { headers: {'X-Token': this.jwt} }).then(response => {
+                    if(response.data.status){
+                        this.added = true;
+                    } else {
+                        this.error = response.data.message;
+                    }
+                })
+                .catch(err => (console.log(err)))
+            }
+        },
+        delete_instance(elem)
+        {
+            axios.get(`/api/opencti/delete/${elem.id}`, { timeout: 10000, headers: {'X-Token': this.jwt} })
+            .then(response => {
+                if(response.data.status){
+                    this.instances = this.instances.filter(function(el) { return el != elem; }); 
+                }
+            })
+            .catch(err => (console.log(err)))
+        },
+        get_opencti_instances()
+        {
+            this.loading = true;
+            this.instances = []
+            axios.get(`/api/opencti/get_all`, { timeout: 10000, headers: {'X-Token': this.jwt} })
+            .then(response => {
+                if(response.data.results){
+                    this.instances = response.data.results;
+                    this.instances.forEach(e => { 
+                        var lastsync = parseInt((Date.now()/1000 - e.lastsync) / 86400)
+                        e.lastsync = (!lastsync)? "Synchronized today" : `Synchronized ${lastsync} day(s) ago`
+                        } )
+                }
+                this.loading = false
+            })
+            .catch(err => (console.log(err)))
+        },
+        switch_tab: function(tab) {
+
+            Object.keys(this.tabs).forEach(key => {
+                if( key == tab ){
+                    this.tabs[key] = true
+                    if (key == "instances") this.get_opencti_instances();
+                } else {
+                    this.tabs[key] = false
+                }
+            });
+        },
+        get_jwt(){
+            axios.get(`/api/get-token`, { timeout: 10000 })
+                .then(response => {
+                    if(response.data.token){
+                        this.jwt = response.data.token
+                    }
+                })
+            .catch(err => (console.log(err)))
+        }
+    },
+    created: function() {
+        this.get_jwt();
+    }
+}
+</script>

assets/requirements.txt

@@ -3,6 +3,7 @@ M2Crypto
 pyOpenSSL
 pydig
 pymisp
+pycti
 netaddr
 pyyaml
 flask

assets/scheme.sql

@@ -28,3 +28,14 @@ CREATE TABLE "misp" (
 	"last_sync" NUMERIC NOT NULL DEFAULT 0,
 	PRIMARY KEY("id" AUTOINCREMENT)
 );
+
+CREATE TABLE "octi" (
+	"id" 	INTEGER UNIQUE,
+	"name" 	TEXT,
+	"url"	TEXT NOT NULL,
+	"apikey"	TEXT NOT NULL,
+	"verifycert" INTEGER NOT NULL DEFAULT 0,
+	"added_on" NUMERIC NOT NULL,
+	"last_sync" NUMERIC NOT NULL DEFAULT 0,
+	PRIMARY KEY("id" AUTOINCREMENT)
+);

server/backend/app/blueprints/octi.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from flask import Blueprint, jsonify, Response, request
+from app.decorators import require_header_token, require_get_token
+from app.classes.octi import OCTI
+
+import json
+
+octi_bp = Blueprint("octi", __name__)
+octi = OCTI()
+
+
+@octi_bp.route('/add', methods=['POST'])
+@require_header_token
+def add_instance():
+    """
+        Parse and add a OpenCTI instance to the database.
+        :return: status of the operation in JSON
+    """
+    data = json.loads(request.data)
+    res = octi.add_instance(data["data"]["instance"])
+    return jsonify(res)
+
+
+@octi_bp.route('/delete/<octi_id>', methods=['GET'])
+@require_header_token
+def delete_instance(octi_id):
+    """
+        Delete a OpenCTI instance by its id to the database.
+        :return: status of the operation in JSON
+    """
+    res = octi.delete_instance(octi_id)
+    return jsonify(res)
+
+
+@octi_bp.route('/get_all', methods=['GET'])
+@require_header_token
+def get_all():
+    """
+        Retreive a list of all OpenCTI instances.
+        :return: list of OpenCTI instances in JSON.
+    """
+    res = octi.get_instances()
+    return jsonify({"results": [i for i in res]})

server/backend/app/classes/octi.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from app import db
+from app.db.models import OCTIInst
+from app.definitions import definitions as defs
+
+from sqlalchemy.sql import exists
+from urllib.parse import unquote
+from flask import escape
+from pycti import OpenCTIApiClient, Infrastructure
+import re
+import time
+import sys
+
+
+class OCTI(object):
+    def __init__(self):
+        return None
+
+    def add_instance(self, instance):
+        """
+            Parse and add a OpenCTI instance to the database.
+            :return: status of the operation in JSON
+        """
+
+        url = instance["url"]
+        name = instance["name"]
+        apikey = instance["key"]
+        verify = instance["ssl"]
+        last_sync = int(time.time()-31536000)  # One year
+
+        sameinstances = db.session.query(OCTIInst).filter(
+            OCTIInst.url == url, OCTIInst.apikey == apikey)
+        if sameinstances.count():
+            return {"status": False,
+                    "message": "This OpenCTI instance already exists"}
+        if name:
+            if self.test_instance(url, apikey, verify):
+                added_on = int(time.time())
+                db.session.add(OCTIInst(name, escape(
+                    url), apikey, verify, added_on, last_sync))
+                db.session.commit()
+                return {"status": True,
+                        "message": "OpenCTI instance added"}
+            else:
+                return {"status": False,
+                        "message": "Please verify the connection to the OpenCTI instance"}
+        else:
+            return {"status": False,
+                    "message": "Please provide a name for your instance"}
+
+    @staticmethod
+    def delete_instance(opencti_id):
+        """
+            Delete a OpenCTI instance by its id in the database.
+            :return: status of the operation in JSON
+        """
+        if db.session.query(exists().where(OCTIInst.id == opencti_id)).scalar():
+            db.session.query(OCTIInst).filter_by(id=opencti_id).delete()
+            db.session.commit()
+            return {"status": True,
+                    "message": "OpenCTI instance deleted"}
+        else:
+            return {"status": False,
+                    "message": "OpenCTI instance not found"}
+
+    def get_instances(self):
+        """
+            Get OpenCTI instances from the database
+            :return: generator of the records.
+        """
+        for opencti in db.session.query(OCTIInst).all():
+            opencti = opencti.__dict__
+            yield {"id": opencti["id"],
+                   "name": opencti["name"],
+                   "url": opencti["url"],
+                   "apikey": opencti["apikey"],
+                   "verifycert": True if opencti["verifycert"] else False,
+                   "connected": self.test_instance(opencti["url"], opencti["apikey"], opencti["verifycert"]),
+                   "lastsync": opencti["last_sync"]}
+
+    @staticmethod
+    def test_instance(url, apikey, verify):
+        """
+            Test the connection of the OpenCTI instance.
+            :return: generator of the records.
+        """
+        try:
+            OpenCTIApiClient(url, token=apikey, ssl_verify=verify)
+            return True
+        except:
+            return False
+
+    @staticmethod
+    def update_sync(opencti_id):
+        """
+            Update the last synchronization date by the actual date.
+            :return: bool, True if updated.
+        """
+        try:
+            misp = OCTIInst.query.get(int(opencti_id))
+            misp.last_sync = int(time.time())
+            db.session.commit()
+            return True
+        except:
+            return False
+
+    @staticmethod
+    def get_iocs(opencti_id):
+        """
+            Get all IOCs from specific OpenCTI instance
+            :return: generator containing the IOCs.
+        """
+        opencti = OCTIInst.query.get(int(opencti_id))
+        if opencti is not None:
+            if opencti.url and opencti.apikey:
+                try:
+                    # Connect to OpenCTI instance and get network activity attributes.
+                    i = OpenCTIApiClient(
+                        opencti.url, opencti.apikey, opencti.verifycert)
+                    r = Infrastructure(i).list(getAll=True)
+                except:
+                    print(
+                        "Unable to connect to the OpenCTI instance ({}/{}).".format(opencti.url, opencti.apikey))
+                    return []
+                """
+                for attr in r["Attribute"]:
+                    if attr["type"] in ["ip-dst", "domain", "snort", "x509-fingerprint-sha1"]:
+
+                        ioc = {"value": attr["value"],
+                               "type": None,
+                               "tag": "suspect",
+                               "tlp": "white"}
+
+                        # Deduce the IOC type.
+                        if re.match(defs["iocs_types"][0]["regex"], attr["value"]):
+                            ioc["type"] = "ip4addr"
+                        elif re.match(defs["iocs_types"][1]["regex"], attr["value"]):
+                            ioc["type"] = "ip6addr"
+                        elif re.match(defs["iocs_types"][2]["regex"], attr["value"]):
+                            ioc["type"] = "cidr"
+                        elif re.match(defs["iocs_types"][3]["regex"], attr["value"]):
+                            ioc["type"] = "domain"
+                        elif re.match(defs["iocs_types"][4]["regex"], attr["value"]):
+                            ioc["type"] = "sha1cert"
+                        elif "alert " in attr["value"][0:6]:
+                            ioc["type"] = "snort"
+                        else:
+                            continue
+
+                        if "Tag" in attr:
+                            for tag in attr["Tag"]:
+                                # Add a TLP to the IOC if defined in tags.
+                                tlp = re.search(
+                                    r"^(?:tlp:)(red|green|amber|white)", tag['name'].lower())
+                                if tlp:
+                                    ioc["tlp"] = tlp.group(1)
+
+                                # Add possible tag (need to match TinyCheck tags)
+                                if tag["name"].lower() in [t["tag"] for t in defs["iocs_tags"]]:
+                                    ioc["tag"] = tag["name"].lower()
+                        yield ioc
+                """

server/backend/app/db/models.py

@@ -29,6 +29,17 @@ class MISPInst(db.Model):
         self.last_sync = last_sync
 
 
+class OCTIInst(db.Model):
+    def __init__(self, name, url, key, ssl, added_on, last_sync):
+        self.name = name
+        self.url = url
+        self.apikey = key
+        self.verifycert = ssl
+        self.added_on = added_on
+        self.last_sync = last_sync
+
+
 db.mapper(Whitelist, db.Table('whitelist', db.metadata, autoload=True))
 db.mapper(Ioc, db.Table('iocs', db.metadata, autoload=True))
 db.mapper(MISPInst, db.Table('misp', db.metadata, autoload=True))
+db.mapper(OCTIInst, db.Table('octi', db.metadata, autoload=True))

server/backend/main.py

@@ -7,6 +7,7 @@ from app.blueprints.ioc import ioc_bp
 from app.blueprints.whitelist import whitelist_bp
 from app.blueprints.config import config_bp
 from app.blueprints.misp import misp_bp
+from app.blueprints.octi import octi_bp
 import datetime
 import secrets
 import jwt
@@ -58,6 +59,7 @@ app.register_blueprint(ioc_bp, url_prefix='/api/ioc')
 app.register_blueprint(whitelist_bp, url_prefix='/api/whitelist')
 app.register_blueprint(config_bp, url_prefix='/api/config')
 app.register_blueprint(misp_bp, url_prefix='/api/misp')
+app.register_blueprint(octi_bp, url_prefix='/api/opencti')
 
 if __name__ == '__main__':
     ssl_cert = "{}/{}".format(path[0], 'cert.pem')

server/backend/watchers.py

@@ -5,6 +5,7 @@ from app.utils import read_config
 from app.classes.iocs import IOCs
 from app.classes.whitelist import WhiteList
 from app.classes.misp import MISP
+from app.classes.octi import OCTI
 
 import requests
 import json
@@ -41,8 +42,10 @@ def watch_iocs():
                     res = requests.get(w["url"], verify=False)
                     if res.status_code == 200:
                         content = json.loads(res.content)
-                        iocs_list = content["iocs"] if "iocs" in content else []
+                        iocs_list = content["iocs"] if "iocs" in content else [
-                        to_delete = content["to_delete"] if "to_delete" in content else []
+                        ]
+                        to_delete = content["to_delete"] if "to_delete" in content else [
+                        ]
                     else:
                         w["status"] = False
                 except:
@@ -89,8 +92,10 @@ def watch_whitelists():
                     res = requests.get(w["url"], verify=False)
                     if res.status_code == 200:
                         content = json.loads(res.content)
-                        elements = content["elements"] if "elements" in content else []
+                        elements = content["elements"] if "elements" in content else [
-                        to_delete = content["to_delete"] if "to_delete" in content else []
+                        ]
+                        to_delete = content["to_delete"] if "to_delete" in content else [
+                        ]
                     else:
                         w["status"] = False
                 except:
@@ -135,13 +140,40 @@ def watch_misp():
                              ioc["value"], "misp-{}".format(ist["id"]))
                 misp.update_sync(ist["id"])
                 instances.pop(i)
-        if instances: time.sleep(60)
+        if instances:
+            time.sleep(60)
+
+
+def watch_opencti():
+    """
+        Retrieve IOCs from OpenCTI instances. Each new element is
+        tested and then added to the database.
+    """
+    iocs, octi = IOCs(), OCTI()
+    instances = [i for i in octi.get_instances()]
+
+    while instances:
+        for i, ist in enumerate(instances):
+            status = octi.test_instance(ist["url"],
+                                        ist["apikey"],
+                                        ist["verifycert"])
+            if status:
+                print("Testing...")
+                # for ioc in octi.get_iocs(ist["id"]):
+                #    iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
+                #             ioc["value"], "octi-{}".format(ist["id"]))
+                # octi.update_sync(ist["id"])
+                instances.pop(i)
+        if instances:
+            time.sleep(60)
 
 
 p1 = Process(target=watch_iocs)
 p2 = Process(target=watch_whitelists)
 p3 = Process(target=watch_misp)
+p4 = Process(target=watch_octi)
 
 p1.start()
 p2.start()
 p3.start()
+p4.start()