Add timeout to requests calls

main

.github/workflows/snorkell-auto-documentation.yml

@@ -0,0 +1,19 @@
+# This workflow will improvise current file with AI genereated documentation and Create new PR
+
+name: Snorkell.ai - Revolutionizing Documentation on GitHub
+
+on:
+  push:
+    branches: ["main"]
+  workflow_dispatch:
+
+jobs:
+  Documentation:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Snorkell DocGen Client
+      uses: SingularityX-ai/snorkell-documentation-client@v1.0.0
+      with:
+        client_id: ${{ secrets.SNORKELL_CLIENT_ID }}
+        api_key: ${{ secrets.SNORKELL_API_KEY }}
+        branch_name: "main"

analysis/classes/zeekengine.py

@@ -270,7 +270,7 @@ class ZeekEngine(object):
                 if record is not None:
                     f = {"filename": record["filename"],
                          "ip_src": record["id.orig_h"],
-                         "ip_dst": record["id.orig_p"],
+                         "ip_dst": record["id.resp_h"],
                          "mime_type": record["mime_type"],
                          "sha1": record["sha1"]}
                     if f not in self.files:

app/frontend/src/assets/icon_plug_usb.svg

@@ -1,4 +1 @@
-<svg width="112" height="195" viewBox="0 0 112 195" fill="none" xmlns="http://www.w3.org/2000/svg">
-<line x1="3.5" y1="3.5" x2="3.50001" y2="191.5" stroke="black" stroke-width="7" stroke-linecap="round" stroke-linejoin="round"/>
-<rect x="7" width="105" height="195" fill="#F7F8F9"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="112" height="195" fill="none" viewBox="0 0 112 195"><line x1="3.5" x2="3.5" y1="3.5" y2="191.5" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="7"/><rect width="105" height="195" x="7" fill="#F7F8F9"/></svg>

app/frontend/src/assets/icon_spinner.svg

@@ -1,16 +1 @@
-<svg version="1.1" id="loader-1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-   width="40px" height="40px" viewBox="0 0 40 40" enable-background="new 0 0 40 40" xml:space="preserve">
-  <path opacity="0.2" fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946
-    s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634
-    c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z"/>
-  <path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0
-    C22.32,8.481,24.301,9.057,26.013,10.047z">
-    <animateTransform attributeType="xml"
-      attributeName="transform"
-      type="rotate"
-      from="0 20 20"
-      to="360 20 20"
-      dur="0.5s"
-      repeatCount="indefinite"/>
-    </path>
-  </svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="loader-1" width="40" height="40" x="0" y="0" enable-background="new 0 0 40 40" version="1.1" viewBox="0 0 40 40" xml:space="preserve"><path fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946 s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634 c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z" opacity=".2"/><path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0 C22.32,8.481,24.301,9.057,26.013,10.047z"><animateTransform attributeName="transform" attributeType="xml" dur="0.5s" from="0 20 20" repeatCount="indefinite" to="360 20 20" type="rotate"/></path></svg>

app/frontend/src/assets/icon_success.svg

@@ -1,5 +1 @@
-<svg width="106" height="106" viewBox="0 0 106 106" fill="none" xmlns="http://www.w3.org/2000/svg">
-<circle cx="53" cy="53" r="53" fill="#40D8A1"/>
-<path d="M29 52.5L47.5 70.5" stroke="white" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"/>
-<line x1="79" y1="40.0711" x2="48.0711" y2="71" stroke="white" stroke-width="10" stroke-linecap="round"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="106" height="106" fill="none" viewBox="0 0 106 106"><circle cx="53" cy="53" r="53" fill="#40D8A1"/><path stroke="#fff" stroke-linecap="round" stroke-linejoin="round" stroke-width="10" d="M29 52.5L47.5 70.5"/><line x1="79" x2="48.071" y1="40.071" y2="71" stroke="#fff" stroke-linecap="round" stroke-width="10"/></svg>

app/frontend/src/assets/icon_usb.svg

@@ -1,6 +1 @@
-<svg width="548" height="199" viewBox="0 0 548 199" fill="none" xmlns="http://www.w3.org/2000/svg">
-<rect x="403" y="27" width="142" height="145" rx="8" fill="white" stroke="black" stroke-width="6" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z" fill="black"/>
-<rect x="477" y="55" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
-<rect x="477" y="117" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="548" height="199" fill="none" viewBox="0 0 548 199"><rect width="142" height="145" x="403" y="27" fill="#fff" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="6" rx="8"/><path fill="#000" d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z"/><rect width="26" height="26" x="477" y="55" fill="#fff" stroke="#000" stroke-width="6"/><rect width="26" height="26" x="477" y="117" fill="#fff" stroke="#000" stroke-width="6"/></svg>

app/frontend/src/assets/loading.svg

@@ -1,11 +1 @@
-<?xml version="1.0" encoding="utf-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin: auto; background: none; display: block; shape-rendering: auto;" width="200px" height="200px" viewBox="0 0 100 100" preserveAspectRatio="xMidYMid">
-<circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2">
-  <animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
-  <animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
-</circle>
-<circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2">
-  <animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline"></animate>
-  <animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline"></animate>
-</circle>
-<!-- [ldio] generated by https://loading.io/ --></svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin:auto;background:0 0;display:block;shape-rendering:auto" width="200" height="200" preserveAspectRatio="xMidYMid" viewBox="0 0 100 100"><circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2"><animate attributeName="r" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle><circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2"><animate attributeName="r" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle></svg>

assets/requirements.txt

@@ -2,9 +2,9 @@ pymisp==2.4.165.1
 sqlalchemy==1.4.48
 ipwhois==1.2.0
 netaddr==0.8.0
-flask==1.1.2
+flask==2.2.5
 flask_httpauth==4.8.0
-pyjwt==1.7.1
+pyjwt==2.4.0
 psutil==5.8.0
 pydig==0.4.0
 pyudev==0.24.0

server/backend/watchers.py

@@ -1,149 +1,149 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from app.utils import read_config
-from app.classes.iocs import IOCs
-from app.classes.whitelist import WhiteList
-from app.classes.misp import MISP
-
-import requests
-import json
-import urllib3
-import time
-from multiprocessing import Process
-
-"""
-    This file is parsing the watchers present
-    in the configuration file. This in order to get
-    automatically new iocs / elements from remote
-    sources without user interaction.
-"""
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-
-def watch_iocs():
-    """
-        Retrieve IOCs from the remote URLs defined in config/watchers.
-        For each IOC, add it to the DB.
-    """
-
-    # Retrieve the URLs from the configuration
-    urls = read_config(("watchers", "iocs"))
-    watchers = [{"url": url, "status": False} for url in urls]
-
-    while True:
-        for w in watchers:
-            if w["status"] == False:
-                iocs = IOCs()
-                iocs_list = []
-                to_delete = []
-                try:
-                    res = requests.get(w["url"], verify=False)
-                    if res.status_code == 200:
-                        content = json.loads(res.content)
-                        iocs_list = content["iocs"] if "iocs" in content else []
-                        to_delete = content["to_delete"] if "to_delete" in content else []
-                    else:
-                        w["status"] = False
-                except:
-                    w["status"] = False
-
-                for ioc in iocs_list:
-                    try:
-                        iocs.add(ioc["type"], ioc["tag"],
-                                 ioc["tlp"], ioc["value"], "watcher")
-                        w["status"] = True
-                    except:
-                        continue
-
-                for ioc in to_delete:
-                    try:
-                        iocs.delete_by_value(ioc["value"])
-                        w["status"] = True
-                    except:
-                        continue
-
-        # If at least one URL haven't be parsed, let's retry in 1min.
-        if False in [w["status"] for w in watchers]:
-            time.sleep(60)
-        else:
-            break
-
-
-def watch_whitelists():
-    """
-        Retrieve whitelist elements from the remote URLs
-        defined in config/watchers. For each (new ?) element,
-        add it to the DB.
-    """
-
-    urls = read_config(("watchers", "whitelists"))
-    watchers = [{"url": url, "status": False} for url in urls]
-
-    while True:
-        for w in watchers:
-            if w["status"] == False:
-                whitelist = WhiteList()
-                elements = []
-                to_delete = []
-                try:
-                    res = requests.get(w["url"], verify=False)
-                    if res.status_code == 200:
-                        content = json.loads(res.content)
-                        elements = content["elements"] if "elements" in content else []
-                        to_delete = content["to_delete"] if "to_delete" in content else []
-                    else:
-                        w["status"] = False
-                except:
-                    w["status"] = False
-
-                for elem in elements:
-                    try:
-                        whitelist.add(elem["type"], elem["element"], "watcher")
-                        w["status"] = True
-                    except:
-                        continue
-
-                for elem in to_delete:
-                    try:
-                        whitelist.delete_by_value(elem["element"])
-                        w["status"] = True
-                    except:
-                        continue
-
-        if False in [w["status"] for w in watchers]:
-            time.sleep(60)
-        else:
-            break
-
-
-def watch_misp():
-    """
-        Retrieve IOCs from misp instances. Each new element is
-        tested and then added to the database.
-    """
-    iocs, misp = IOCs(), MISP()
-    instances = [i for i in misp.get_instances()]
-
-    while instances:
-        for i, ist in enumerate(instances):
-            status = misp.test_instance(ist["url"],
-                                        ist["apikey"],
-                                        ist["verifycert"])
-            if status:
-                for ioc in misp.get_iocs(ist["id"]):
-                    iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
-                             ioc["value"], "misp-{}".format(ist["id"]))
-                misp.update_sync(ist["id"])
-                instances.pop(i)
-        if instances: time.sleep(60)
-
-
-p1 = Process(target=watch_iocs)
-p2 = Process(target=watch_whitelists)
-p3 = Process(target=watch_misp)
-
-p1.start()
-p2.start()
-p3.start()
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from app.utils import read_config
+from app.classes.iocs import IOCs
+from app.classes.whitelist import WhiteList
+from app.classes.misp import MISP
+
+import requests
+import json
+import urllib3
+import time
+from multiprocessing import Process
+
+"""
+    This file is parsing the watchers present
+    in the configuration file. This in order to get
+    automatically new iocs / elements from remote
+    sources without user interaction.
+"""
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def watch_iocs():
+    """
+        Retrieve IOCs from the remote URLs defined in config/watchers.
+        For each IOC, add it to the DB.
+    """
+
+    # Retrieve the URLs from the configuration
+    urls = read_config(("watchers", "iocs"))
+    watchers = [{"url": url, "status": False} for url in urls]
+
+    while True:
+        for w in watchers:
+            if w["status"] == False:
+                iocs = IOCs()
+                iocs_list = []
+                to_delete = []
+                try:
+                    res = requests.get(w["url"], verify=False, timeout=60)
+                    if res.status_code == 200:
+                        content = json.loads(res.content)
+                        iocs_list = content["iocs"] if "iocs" in content else []
+                        to_delete = content["to_delete"] if "to_delete" in content else []
+                    else:
+                        w["status"] = False
+                except:
+                    w["status"] = False
+
+                for ioc in iocs_list:
+                    try:
+                        iocs.add(ioc["type"], ioc["tag"],
+                                 ioc["tlp"], ioc["value"], "watcher")
+                        w["status"] = True
+                    except:
+                        continue
+
+                for ioc in to_delete:
+                    try:
+                        iocs.delete_by_value(ioc["value"])
+                        w["status"] = True
+                    except:
+                        continue
+
+        # If at least one URL haven't be parsed, let's retry in 1min.
+        if False in [w["status"] for w in watchers]:
+            time.sleep(60)
+        else:
+            break
+
+
+def watch_whitelists():
+    """
+        Retrieve whitelist elements from the remote URLs
+        defined in config/watchers. For each (new ?) element,
+        add it to the DB.
+    """
+
+    urls = read_config(("watchers", "whitelists"))
+    watchers = [{"url": url, "status": False} for url in urls]
+
+    while True:
+        for w in watchers:
+            if w["status"] == False:
+                whitelist = WhiteList()
+                elements = []
+                to_delete = []
+                try:
+                    res = requests.get(w["url"], verify=False, timeout=60)
+                    if res.status_code == 200:
+                        content = json.loads(res.content)
+                        elements = content["elements"] if "elements" in content else []
+                        to_delete = content["to_delete"] if "to_delete" in content else []
+                    else:
+                        w["status"] = False
+                except:
+                    w["status"] = False
+
+                for elem in elements:
+                    try:
+                        whitelist.add(elem["type"], elem["element"], "watcher")
+                        w["status"] = True
+                    except:
+                        continue
+
+                for elem in to_delete:
+                    try:
+                        whitelist.delete_by_value(elem["element"])
+                        w["status"] = True
+                    except:
+                        continue
+
+        if False in [w["status"] for w in watchers]:
+            time.sleep(60)
+        else:
+            break
+
+
+def watch_misp():
+    """
+        Retrieve IOCs from misp instances. Each new element is
+        tested and then added to the database.
+    """
+    iocs, misp = IOCs(), MISP()
+    instances = [i for i in misp.get_instances()]
+
+    while instances:
+        for i, ist in enumerate(instances):
+            status = misp.test_instance(ist["url"],
+                                        ist["apikey"],
+                                        ist["verifycert"])
+            if status:
+                for ioc in misp.get_iocs(ist["id"]):
+                    iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
+                             ioc["value"], "misp-{}".format(ist["id"]))
+                misp.update_sync(ist["id"])
+                instances.pop(i)
+        if instances: time.sleep(60)
+
+
+p1 = Process(target=watch_iocs)
+p2 = Process(target=watch_whitelists)
+p3 = Process(target=watch_misp)
+
+p1.start()
+p2.start()
+p3.start()

server/frontend/app/classes/update.py

@@ -1,82 +1,82 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from app.utils import read_config
-import subprocess as sp
-import requests
-import json
-import os
-import re
-
-
-class Update(object):
-
-    def __init__(self):
-        self.project_url = "https://api.github.com/repos/KasperskyLab/TinyCheck/tags"
-        self.app_path = "/usr/share/tinycheck"
-        return None
-
-    def check_version(self):
-        """
-            Check if a new version of TinyCheck is available 
-            by quering the Github api and comparing the last
-            tag inside the VERSION file.
-            :return: dict containing the available versions.
-        """
-        if read_config(("frontend", "update")):
-            try:
-                res = requests.get(self.project_url)
-                res = json.loads(res.content.decode("utf8"))
-
-                with open(os.path.join(self.app_path, "VERSION")) as f:
-                    cv = f.read()
-                    if cv != res[0]["name"]:
-                        return {"status": True,
-                                "message": "A new version is available",
-                                "current_version": cv,
-                                "next_version": res[0]["name"]}
-                    else:
-                        return {"status": True,
-                                "message": "This is the latest version",
-                                "current_version": cv}
-            except:
-                return {"status": False,
-                        "message": "Something went wrong (no API access nor version file)"}
-        else:
-            return {"status": False,
-                    "message": "You don't have rights to do this operation."}
-
-    def get_current_version(self):
-        """
-            Get the current version of the TinyCheck instance
-            :return: dict containing the current version or error.
-        """
-        if read_config(("frontend", "update")):
-            try:
-                with open(os.path.join(self.app_path, "VERSION")) as f:
-                    return {"status": True,
-                            "current_version": f.read()}
-            except:
-                return {"status": False,
-                        "message": "Something went wrong - no version file ?"}
-        else:
-            return {"status": False,
-                    "message": "You don't have rights to do this operation."}
-
-    def update_instance(self):
-        """
-            Update the instance by executing the update script.
-            :return: dict containing the update status.
-        """
-        if read_config(("frontend", "update")):
-            try:
-                os.chdir(self.app_path)
-                sp.Popen(["bash", os.path.join(self.app_path, "update.sh")])
-                return {"status": True,
-                        "message": "Update successfully launched"}
-            except:
-                return {"status": False,
-                        "message": "Issue during the update"}
-        else:
-            return {"status": False,
-                    "message": "You don't have rights to do this operation."}
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from app.utils import read_config
+import subprocess as sp
+import requests
+import json
+import os
+import re
+
+
+class Update(object):
+
+    def __init__(self):
+        self.project_url = "https://api.github.com/repos/KasperskyLab/TinyCheck/tags"
+        self.app_path = "/usr/share/tinycheck"
+        return None
+
+    def check_version(self):
+        """
+            Check if a new version of TinyCheck is available 
+            by quering the Github api and comparing the last
+            tag inside the VERSION file.
+            :return: dict containing the available versions.
+        """
+        if read_config(("frontend", "update")):
+            try:
+                res = requests.get(self.project_url, timeout=60)
+                res = json.loads(res.content.decode("utf8"))
+
+                with open(os.path.join(self.app_path, "VERSION")) as f:
+                    cv = f.read()
+                    if cv != res[0]["name"]:
+                        return {"status": True,
+                                "message": "A new version is available",
+                                "current_version": cv,
+                                "next_version": res[0]["name"]}
+                    else:
+                        return {"status": True,
+                                "message": "This is the latest version",
+                                "current_version": cv}
+            except:
+                return {"status": False,
+                        "message": "Something went wrong (no API access nor version file)"}
+        else:
+            return {"status": False,
+                    "message": "You don't have rights to do this operation."}
+
+    def get_current_version(self):
+        """
+            Get the current version of the TinyCheck instance
+            :return: dict containing the current version or error.
+        """
+        if read_config(("frontend", "update")):
+            try:
+                with open(os.path.join(self.app_path, "VERSION")) as f:
+                    return {"status": True,
+                            "current_version": f.read()}
+            except:
+                return {"status": False,
+                        "message": "Something went wrong - no version file ?"}
+        else:
+            return {"status": False,
+                    "message": "You don't have rights to do this operation."}
+
+    def update_instance(self):
+        """
+            Update the instance by executing the update script.
+            :return: dict containing the update status.
+        """
+        if read_config(("frontend", "update")):
+            try:
+                os.chdir(self.app_path)
+                sp.Popen(["bash", os.path.join(self.app_path, "update.sh")])
+                return {"status": True,
+                        "message": "Update successfully launched"}
+            except:
+                return {"status": False,
+                        "message": "Issue during the update"}
+        else:
+            return {"status": False,
+                    "message": "You don't have rights to do this operation."}