Limit readline()

main

.github/workflows/snorkell-auto-documentation.yml

@@ -0,0 +1,19 @@
+# This workflow will improvise current file with AI genereated documentation and Create new PR
+
+name: Snorkell.ai - Revolutionizing Documentation on GitHub
+
+on:
+  push:
+    branches: ["main"]
+  workflow_dispatch:
+
+jobs:
+  Documentation:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Snorkell DocGen Client
+      uses: SingularityX-ai/snorkell-documentation-client@v1.0.0
+      with:
+        client_id: ${{ secrets.SNORKELL_CLIENT_ID }}
+        api_key: ${{ secrets.SNORKELL_API_KEY }}
+        branch_name: "main"

analysis/classes/parsezeeklogs.py

@@ -1,178 +1,178 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from json import loads, dumps
-from collections import OrderedDict
-from datetime import datetime
-from traceback import print_exc
-
-# Taken from https://github.com/dgunter/ParseZeekLogs <3
-
-
-class ParseZeekLogs(object):
-    """
-        Class that parses Zeek logs and allows log data to be output in CSV or json format.
-        Attributes: filepath: Path of Zeek log file to read
-    """
-
-    def __init__(self, filepath, batchsize=500, fields=None, output_format=None, ignore_keys=[], meta={}, safe_headers=False):
-        self.fd = open(filepath, "r")
-        self.options = OrderedDict()
-        self.firstRun = True
-        self.filtered_fields = fields
-        self.batchsize = batchsize
-        self.output_format = output_format
-        self.ignore_keys = ignore_keys
-        self.meta = meta
-        self.safe_headers = safe_headers
-
-        # Convert ' to " in meta string
-        meta = loads(dumps(meta).replace("'", '"'))
-
-        # Read the header option lines
-        l = self.fd.readline().strip()
-        while l.strip().startswith("#"):
-            # Parse the options out
-            if l.startswith("#separator"):
-                key = str(l[1:].split(" ")[0])
-                value = str.encode(l[1:].split(
-                    " ")[1].strip()).decode('unicode_escape')
-                self.options[key] = value
-            elif l.startswith("#"):
-                key = str(l[1:].split(self.options.get('separator'))[0])
-                value = l[1:].split(self.options.get('separator'))[1:]
-                self.options[key] = value
-
-            # Read the next line
-            l = self.fd.readline().strip()
-
-        self.firstLine = l
-
-        # Save mapping of fields to values:
-        self.fields = self.options.get('fields')
-        self.types = self.options.get('types')
-
-        self.data_types = {}
-        for i, val in enumerate(self.fields):
-            # Convert field names if safe_headers is enabled
-            if self.safe_headers is True:
-                self.fields[i] = self.fields[i].replace(".", "_")
-
-            # Match types with each other
-            self.data_types[self.fields[i]] = self.types[i]
-
-    def __del__(self):
-        self.fd.close()
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        retVal = ""
-        if self.firstRun is True:
-            retVal = self.firstLine
-            self.firstRun = False
-        else:
-            retVal = self.fd.readline().strip()
-
-        # If an empty string is returned, readline is done reading
-        if retVal == "" or retVal is None:
-            raise StopIteration
-
-        # Split out the data we are going to return
-        retVal = retVal.split(self.options.get('separator'))
-
-        record = None
-        # Make sure we aren't dealing with a comment line
-        if len(retVal) > 0 and not str(retVal[0]).strip().startswith("#") \
-                and len(retVal) is len(self.options.get("fields")):
-            record = OrderedDict()
-            # Prepare fields for conversion
-            for x in range(0, len(retVal)):
-                if self.safe_headers is True:
-                    converted_field_name = self.options.get(
-                        "fields")[x].replace(".", "_")
-                else:
-                    converted_field_name = self.options.get("fields")[x]
-                if self.filtered_fields is None or converted_field_name in self.filtered_fields:
-                    # Translate - to "" to fix a conversation error
-                    if retVal[x] == "-":
-                        retVal[x] = ""
-                    # Save the record field if the field isn't filtered out
-                    record[converted_field_name] = retVal[x]
-
-            # Convert values to the appropriate record type
-            record = self.convert_values(
-                record, self.ignore_keys, self.data_types)
-
-            if record is not None and self.output_format == "json":
-                # Output will be json
-
-                # Add metadata to json
-                for k, v in self.meta.items():
-                    record[k] = v
-
-                retVal = record
-            elif record is not None and self.output_format == "csv":
-                retVal = ""
-                # Add escaping to csv format
-                for k, v in record.items():
-                    # Add escaping to string values
-                    if isinstance(v, str):
-                        retVal += str("\"" + str(v).strip() + "\"" + ",")
-                    else:
-                        retVal += str(str(v).strip() + ",")
-                # Remove the trailing comma
-                retVal = retVal[:-1]
-        else:
-            retVal = None
-
-        return retVal
-
-    def convert_values(self, data, ignore_keys=[], data_types={}):
-        keys_to_delete = []
-        for k, v in data.items():
-            # print("evaluating k: " + str(k) + " v: " + str(v))
-
-            if isinstance(v, dict):
-                data[k] = self.convert_values(v)
-            else:
-                if data_types.get(k) is not None:
-                    if (data_types.get(k) == "port" or data_types.get(k) == "count"):
-                        if v != "":
-                            data[k] = int(v)
-                        else:
-                            keys_to_delete.append(k)
-                    elif (data_types.get(k) == "double" or data_types.get(k) == "interval"):
-                        if v != "":
-                            data[k] = float(v)
-                        else:
-                            keys_to_delete.append(k)
-                    elif data_types.get(k) == "bool":
-                        data[k] = bool(v)
-                    else:
-                        data[k] = v
-
-        for k in keys_to_delete:
-            del data[k]
-
-        return data
-
-    def get_fields(self):
-        """Returns all fields present in the log file
-        Returns:
-            A python list containing all field names in the log file
-        """
-        field_names = ""
-        if self.output_format == "csv":
-            for i, v in enumerate(self.fields):
-                if self.filtered_fields is None or v in self.filtered_fields:
-                    field_names += str(v) + ","
-            # Remove the trailing comma
-            field_names = field_names[:-1].strip()
-        else:
-            field_names = []
-            for i, v in enumerate(self.fields):
-                if self.filtered_fields is None or v in self.filtered_fields:
-                    field_names.append(v)
-        return field_names
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from json import loads, dumps
+from collections import OrderedDict
+from datetime import datetime
+from traceback import print_exc
+
+# Taken from https://github.com/dgunter/ParseZeekLogs <3
+
+
+class ParseZeekLogs(object):
+    """
+        Class that parses Zeek logs and allows log data to be output in CSV or json format.
+        Attributes: filepath: Path of Zeek log file to read
+    """
+
+    def __init__(self, filepath, batchsize=500, fields=None, output_format=None, ignore_keys=[], meta={}, safe_headers=False):
+        self.fd = open(filepath, "r")
+        self.options = OrderedDict()
+        self.firstRun = True
+        self.filtered_fields = fields
+        self.batchsize = batchsize
+        self.output_format = output_format
+        self.ignore_keys = ignore_keys
+        self.meta = meta
+        self.safe_headers = safe_headers
+
+        # Convert ' to " in meta string
+        meta = loads(dumps(meta).replace("'", '"'))
+
+        # Read the header option lines
+        l = self.fd.readline(5_000_000).strip()
+        while l.strip().startswith("#"):
+            # Parse the options out
+            if l.startswith("#separator"):
+                key = str(l[1:].split(" ")[0])
+                value = str.encode(l[1:].split(
+                    " ")[1].strip()).decode('unicode_escape')
+                self.options[key] = value
+            elif l.startswith("#"):
+                key = str(l[1:].split(self.options.get('separator'))[0])
+                value = l[1:].split(self.options.get('separator'))[1:]
+                self.options[key] = value
+
+            # Read the next line
+            l = self.fd.readline(5_000_000).strip()
+
+        self.firstLine = l
+
+        # Save mapping of fields to values:
+        self.fields = self.options.get('fields')
+        self.types = self.options.get('types')
+
+        self.data_types = {}
+        for i, val in enumerate(self.fields):
+            # Convert field names if safe_headers is enabled
+            if self.safe_headers is True:
+                self.fields[i] = self.fields[i].replace(".", "_")
+
+            # Match types with each other
+            self.data_types[self.fields[i]] = self.types[i]
+
+    def __del__(self):
+        self.fd.close()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        retVal = ""
+        if self.firstRun is True:
+            retVal = self.firstLine
+            self.firstRun = False
+        else:
+            retVal = self.fd.readline().strip()
+
+        # If an empty string is returned, readline is done reading
+        if retVal == "" or retVal is None:
+            raise StopIteration
+
+        # Split out the data we are going to return
+        retVal = retVal.split(self.options.get('separator'))
+
+        record = None
+        # Make sure we aren't dealing with a comment line
+        if len(retVal) > 0 and not str(retVal[0]).strip().startswith("#") \
+                and len(retVal) is len(self.options.get("fields")):
+            record = OrderedDict()
+            # Prepare fields for conversion
+            for x in range(0, len(retVal)):
+                if self.safe_headers is True:
+                    converted_field_name = self.options.get(
+                        "fields")[x].replace(".", "_")
+                else:
+                    converted_field_name = self.options.get("fields")[x]
+                if self.filtered_fields is None or converted_field_name in self.filtered_fields:
+                    # Translate - to "" to fix a conversation error
+                    if retVal[x] == "-":
+                        retVal[x] = ""
+                    # Save the record field if the field isn't filtered out
+                    record[converted_field_name] = retVal[x]
+
+            # Convert values to the appropriate record type
+            record = self.convert_values(
+                record, self.ignore_keys, self.data_types)
+
+            if record is not None and self.output_format == "json":
+                # Output will be json
+
+                # Add metadata to json
+                for k, v in self.meta.items():
+                    record[k] = v
+
+                retVal = record
+            elif record is not None and self.output_format == "csv":
+                retVal = ""
+                # Add escaping to csv format
+                for k, v in record.items():
+                    # Add escaping to string values
+                    if isinstance(v, str):
+                        retVal += str("\"" + str(v).strip() + "\"" + ",")
+                    else:
+                        retVal += str(str(v).strip() + ",")
+                # Remove the trailing comma
+                retVal = retVal[:-1]
+        else:
+            retVal = None
+
+        return retVal
+
+    def convert_values(self, data, ignore_keys=[], data_types={}):
+        keys_to_delete = []
+        for k, v in data.items():
+            # print("evaluating k: " + str(k) + " v: " + str(v))
+
+            if isinstance(v, dict):
+                data[k] = self.convert_values(v)
+            else:
+                if data_types.get(k) is not None:
+                    if (data_types.get(k) == "port" or data_types.get(k) == "count"):
+                        if v != "":
+                            data[k] = int(v)
+                        else:
+                            keys_to_delete.append(k)
+                    elif (data_types.get(k) == "double" or data_types.get(k) == "interval"):
+                        if v != "":
+                            data[k] = float(v)
+                        else:
+                            keys_to_delete.append(k)
+                    elif data_types.get(k) == "bool":
+                        data[k] = bool(v)
+                    else:
+                        data[k] = v
+
+        for k in keys_to_delete:
+            del data[k]
+
+        return data
+
+    def get_fields(self):
+        """Returns all fields present in the log file
+        Returns:
+            A python list containing all field names in the log file
+        """
+        field_names = ""
+        if self.output_format == "csv":
+            for i, v in enumerate(self.fields):
+                if self.filtered_fields is None or v in self.filtered_fields:
+                    field_names += str(v) + ","
+            # Remove the trailing comma
+            field_names = field_names[:-1].strip()
+        else:
+            field_names = []
+            for i, v in enumerate(self.fields):
+                if self.filtered_fields is None or v in self.filtered_fields:
+                    field_names.append(v)
+        return field_names

analysis/classes/zeekengine.py

@@ -270,7 +270,7 @@ class ZeekEngine(object):
                 if record is not None:
                     f = {"filename": record["filename"],
                          "ip_src": record["id.orig_h"],
-                         "ip_dst": record["id.orig_p"],
+                         "ip_dst": record["id.resp_h"],
                          "mime_type": record["mime_type"],
                          "sha1": record["sha1"]}
                     if f not in self.files:

app/frontend/src/assets/icon_plug_usb.svg

@@ -1,4 +1 @@
-<svg width="112" height="195" viewBox="0 0 112 195" fill="none" xmlns="http://www.w3.org/2000/svg">
-<line x1="3.5" y1="3.5" x2="3.50001" y2="191.5" stroke="black" stroke-width="7" stroke-linecap="round" stroke-linejoin="round"/>
-<rect x="7" width="105" height="195" fill="#F7F8F9"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="112" height="195" fill="none" viewBox="0 0 112 195"><line x1="3.5" x2="3.5" y1="3.5" y2="191.5" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="7"/><rect width="105" height="195" x="7" fill="#F7F8F9"/></svg>

app/frontend/src/assets/icon_spinner.svg

@@ -1,16 +1 @@
-<svg version="1.1" id="loader-1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-   width="40px" height="40px" viewBox="0 0 40 40" enable-background="new 0 0 40 40" xml:space="preserve">
-  <path opacity="0.2" fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946
-    s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634
-    c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z"/>
-  <path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0
-    C22.32,8.481,24.301,9.057,26.013,10.047z">
-    <animateTransform attributeType="xml"
-      attributeName="transform"
-      type="rotate"
-      from="0 20 20"
-      to="360 20 20"
-      dur="0.5s"
-      repeatCount="indefinite"/>
-    </path>
-  </svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="loader-1" width="40" height="40" x="0" y="0" enable-background="new 0 0 40 40" version="1.1" viewBox="0 0 40 40" xml:space="preserve"><path fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946 s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634 c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z" opacity=".2"/><path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0 C22.32,8.481,24.301,9.057,26.013,10.047z"><animateTransform attributeName="transform" attributeType="xml" dur="0.5s" from="0 20 20" repeatCount="indefinite" to="360 20 20" type="rotate"/></path></svg>

app/frontend/src/assets/icon_success.svg

@@ -1,5 +1 @@
-<svg width="106" height="106" viewBox="0 0 106 106" fill="none" xmlns="http://www.w3.org/2000/svg">
-<circle cx="53" cy="53" r="53" fill="#40D8A1"/>
-<path d="M29 52.5L47.5 70.5" stroke="white" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"/>
-<line x1="79" y1="40.0711" x2="48.0711" y2="71" stroke="white" stroke-width="10" stroke-linecap="round"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="106" height="106" fill="none" viewBox="0 0 106 106"><circle cx="53" cy="53" r="53" fill="#40D8A1"/><path stroke="#fff" stroke-linecap="round" stroke-linejoin="round" stroke-width="10" d="M29 52.5L47.5 70.5"/><line x1="79" x2="48.071" y1="40.071" y2="71" stroke="#fff" stroke-linecap="round" stroke-width="10"/></svg>

app/frontend/src/assets/icon_usb.svg

@@ -1,6 +1 @@
-<svg width="548" height="199" viewBox="0 0 548 199" fill="none" xmlns="http://www.w3.org/2000/svg">
-<rect x="403" y="27" width="142" height="145" rx="8" fill="white" stroke="black" stroke-width="6" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z" fill="black"/>
-<rect x="477" y="55" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
-<rect x="477" y="117" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" width="548" height="199" fill="none" viewBox="0 0 548 199"><rect width="142" height="145" x="403" y="27" fill="#fff" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="6" rx="8"/><path fill="#000" d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z"/><rect width="26" height="26" x="477" y="55" fill="#fff" stroke="#000" stroke-width="6"/><rect width="26" height="26" x="477" y="117" fill="#fff" stroke="#000" stroke-width="6"/></svg>

app/frontend/src/assets/loading.svg

@@ -1,11 +1 @@
-<?xml version="1.0" encoding="utf-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin: auto; background: none; display: block; shape-rendering: auto;" width="200px" height="200px" viewBox="0 0 100 100" preserveAspectRatio="xMidYMid">
-<circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2">
-  <animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
-  <animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
-</circle>
-<circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2">
-  <animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline"></animate>
-  <animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline"></animate>
-</circle>
-<!-- [ldio] generated by https://loading.io/ --></svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin:auto;background:0 0;display:block;shape-rendering:auto" width="200" height="200" preserveAspectRatio="xMidYMid" viewBox="0 0 100 100"><circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2"><animate attributeName="r" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle><circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2"><animate attributeName="r" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle></svg>

assets/requirements.txt

@@ -1,19 +1,17 @@
-ipwhois
-M2Crypto
-pyOpenSSL
-pydig
-pymisp
-netaddr
-pyyaml
-flask
-flask_httpauth
-pyjwt
-sqlalchemy
-psutil
-pyudev
-wifi
-qrcode
-netifaces
-weasyprint
-python-whois
-six
+pymisp==2.4.165.1
+sqlalchemy==1.4.48
+ipwhois==1.2.0
+netaddr==0.8.0
+flask==2.2.5
+flask_httpauth==4.8.0
+pyjwt==2.4.0
+psutil==5.8.0
+pydig==0.4.0
+pyudev==0.24.0
+pyyaml==5.3.1
+wifi==0.3.8
+qrcode==7.3.1
+netifaces==0.11.0
+weasyprint==59.0
+python-whois==0.8.0
+six==1.16.0

assets/scheme.sql

@@ -1,30 +1,30 @@
 CREATE TABLE "iocs" (
 	"id"	INTEGER UNIQUE,
-	"value"	TEXT NOT NULL,
-	"type"	TEXT NOT NULL,
-	"tlp"	TEXT NOT NULL,
-	"tag"	TEXT NOT NULL,
-	"source"	TEXT NOT NULL,
-	"added_on"	NUMERIC NOT NULL,
+	"value"	   TEXT NOT NULL,
+	"type"	   TEXT NOT NULL,
+	"tlp"	   TEXT NOT NULL,
+	"tag"	   TEXT NOT NULL,
+	"source"   TEXT NOT NULL,
+	"added_on" TEXT NOT NULL,
 	PRIMARY KEY("id" AUTOINCREMENT)
 );
 
 CREATE TABLE "whitelist" (
 	"id"	INTEGER UNIQUE,
-	"element"	TEXT NOT NULL UNIQUE,
-	"type"	TEXT NOT NULL,
-	"source"	TEXT NOT NULL,
-	"added_on"	INTEGER NOT NULL,
+	"element"  TEXT NOT NULL UNIQUE,
+	"type"	   TEXT NOT NULL,
+	"source"   TEXT NOT NULL,
+	"added_on" TEXT NOT NULL,
 	PRIMARY KEY("id" AUTOINCREMENT)
 );
 
 CREATE TABLE "misp" (
-	"id" 	INTEGER UNIQUE,
-	"name" 	TEXT,
-	"url"	TEXT NOT NULL,
-	"apikey"	TEXT NOT NULL,
+	"id" 	 INTEGER UNIQUE,
+	"name" 	     TEXT,
+	"url"	     TEXT NOT NULL,
+	"apikey"     TEXT NOT NULL,
 	"verifycert" INTEGER NOT NULL DEFAULT 0,
-	"added_on" NUMERIC NOT NULL,
-	"last_sync" NUMERIC NOT NULL DEFAULT 0,
+	"added_on"   TEXT NOT NULL,
+	"last_sync"  TEXT NOT NULL DEFAULT 0,
 	PRIMARY KEY("id" AUTOINCREMENT)
 );

install.sh

@@ -69,7 +69,7 @@ set_credentials() {
     read -s password2
     echo ""
 
-    if [ $password1 = $password2 ]; then
+    if [ "$password1" == "$password2" ]; then
         password=$(echo -n "$password1" | sha256sum | cut -d" " -f1)
         sed -i "s/userlogin/$login/g" /usr/share/tinycheck/config.yaml
         sed -i "s/userpassword/$password/g" /usr/share/tinycheck/config.yaml

server/backend/diagnostics.py

@@ -0,0 +1,191 @@
+#!/usr/bin/python
+import os
+import subprocess
+import platform
+import socket
+import pkg_resources
+import psutil
+
+__author__ = 'Eugeny N Ablesov'
+__version__ = '1.0.17'
+
+def collect_accounts_info():
+    """ This call collects generic information about
+        user accounts presented on system running TinyCheck.
+
+        No personal information collected or provided by this call.
+    """
+    accs = { }
+    users = psutil.users()
+    for user in users:
+        accs[user.name + '@' + user.host] = {
+            'started': user.started,
+            'term': user.terminal
+    }
+    alt_user = os.getenv('SUDO_USER', os.getenv('USER'))
+    usr = 'root' if os.path.expanduser('~') == '/root' else alt_user
+    pid = psutil.Process().pid
+    term = psutil.Process().terminal() if 'Linux' in platform.system() else 'win'
+    accs[usr + '@' + term] = { 'pid': pid }
+    return accs
+
+def collect_os_info():
+    """ This call collects generic information about 
+        operating system running TinyCheck.
+
+        No personal information collected or provided by this call.
+    """
+    os_info = { }
+    os_info['system'] = platform.system()
+    os_info['release'] = platform.release()
+    os_info['version'] = platform.version()
+    os_info['platform'] = platform.platform(aliased=True)
+    if 'Windows' in os_info['system']:
+        os_info['dist'] = platform.win32_ver()
+    if 'Linux' in os_info['system']:
+        os_info['dist'] = platform.libc_ver()
+    return os_info
+
+def collect_hardware_info():
+    """ This call collects information about hardware running TinyCheck.
+
+        No personal information collected or provided by this call.
+    """
+    hw_info = { }
+    hw_info['arch'] = platform.architecture()
+    hw_info['machine'] = platform.machine()
+    hw_info['cpus'] = psutil.cpu_count(logical=False)
+    hw_info['cores'] = psutil.cpu_count()
+    hw_info['load'] = psutil.getloadavg()
+    disk_info = psutil.disk_usage('/')
+    hw_info['disk'] = {
+        'total': disk_info.total,
+        'used': disk_info.used,
+        'free': disk_info.free
+    }
+    return hw_info
+
+def collect_network_info():
+    """ This call collects information about 
+        network configuration and state running TinyCheck.
+
+        No personal information collected or provided by this call.
+    """
+    net_info = { }
+    net_info['namei'] = socket.if_nameindex()
+    addrs = psutil.net_if_addrs()
+    state = psutil.net_io_counters(pernic=True)
+    for interface in addrs.keys():
+        net_info[interface] = { }
+        int_info = state[interface]
+        props = [p for p in dir(int_info)
+            if  not p.startswith("_")
+                and not p == "index"
+                and not p == "count"]
+        for prop in props:
+            net_info[interface][prop] = getattr(int_info, prop)
+    return net_info
+
+def collect_dependency_info(package_list):
+    """ This call collects information about
+        python packages required to run TinyCheck.
+    
+        No personal information collected or provided by this call.
+    """
+    dependencies = { }
+    installed_packages = list(pkg_resources.working_set)
+    installed_packages_list = sorted(["%s==%s"
+        % (installed.key, installed.version)
+        for installed in installed_packages])
+    for pkg in installed_packages_list:
+        [package_name, package_version] = pkg.split('==')
+        if package_name in package_list:
+            dependencies[package_name] = package_version
+    return dependencies
+
+def collect_db_tables_records_count(db_path, tables):
+    result = { }
+    for table in tables:
+        query = 'SELECT COUNT(*) FROM %s' % (table)
+        sqlite_call = subprocess.Popen(['sqlite3', db_path, query], stdout = subprocess.PIPE)
+        stout, sterr = sqlite_call.communicate()
+        val = stout.decode("utf-8")
+        recs = int(val) if val else 0
+        result[table] = recs
+    return result
+
+def collect_internal_state(db_path, tables, to_check):
+    """ This call collects information about
+        installed TinyCheck instance and its internal state.
+    
+        No personal information collected or provided by this call.
+    """
+    state_ = { }
+    available = os.path.isfile(db_path)
+    dbsize = 0
+    state_['db'] = {
+        'available': available,
+        'size': dbsize
+    }
+    state_['db']['records'] = { }
+    if available:
+        state_['db']['size'] = os.stat(db_path).st_size
+        state_['db']['records'] = collect_db_tables_records_count(db_path, tables)
+
+    services_ = { }
+    for alias in to_check:
+        status = subprocess.call(['systemctl', 'is-active', '--quiet', '%s' % (to_check[alias])])
+        state = ''
+        if status != 0:
+            sysctl_call = subprocess.Popen(
+                ["systemctl", "status", "%s" % (to_check[alias]),
+                r"|",
+                "grep",
+                r"''"],
+                stdout = subprocess.PIPE,
+                stderr = subprocess.PIPE)
+            stout, sterr = sysctl_call.communicate()
+            state = stout.decode("utf-8")
+            errs = sterr.decode("utf-8")
+            if "could not be found" in errs:
+                state = 'Service not found'
+        services_[alias] = {
+            'running': status == 0,
+            'status': status,
+            'state': state
+        }
+    state_['svc'] = services_
+    return state_
+
+def main():
+    print("TinyCheck diagnostics script.\nVersion: %s" % (__version__))
+    print("")
+
+    db_path = '/usr/share/tinycheck/tinycheck.sqlite3'
+    tables = ['iocs', 'whitelist', 'misp']
+    services = { }
+    services['frontend'] = 'tinycheck-frontend.service'
+    services['backend'] = 'tinycheck-backend.service'
+    services['kiosk'] = 'tinycheck-kiosk.service'
+    services['watchers'] = 'tinycheck-watchers.service'
+
+    deps = [
+        'pymisp', 'sqlalchemy', 'ipwhois',
+        'netaddr', 'flask', 'flask_httpauth',
+        'pyjwt', 'psutil', 'pydig', 'pyudev',
+        'pyyaml', 'wifi', 'qrcode', 'netifaces',
+        'weasyprint', 'python-whois', 'six' ]
+
+    diagnostics = { }
+    diagnostics['acc'] = collect_accounts_info()
+    diagnostics['os'] = collect_os_info()
+    diagnostics['hw'] = collect_hardware_info()
+    diagnostics['net'] = collect_network_info()
+    diagnostics['deps'] = collect_dependency_info(deps)
+    diagnostics['state'] = collect_internal_state(db_path, tables, services)
+    report = { 'diagnostics': diagnostics }
+    print(report)
+    print("")
+
+if __name__ == '__main__':
+    main()

server/backend/main.py

@@ -10,7 +10,6 @@ from app.blueprints.misp import misp_bp
 import datetime
 import secrets
 import jwt
-from OpenSSL import SSL
 from app.utils import read_config
 from sys import path
 
@@ -64,7 +63,6 @@ if __name__ == '__main__':
     ssl_key = "{}/{}".format(path[0], 'key.pem')
 
     if read_config(("backend", "remote_access")):
-        app.run(host="0.0.0.0", port=443,
-                ssl_context=(ssl_cert, ssl_key))
+        app.run(host="0.0.0.0", port=443, ssl_context=(ssl_cert, ssl_key))
     else:
-        app.run(port=443, ssl_context=(ssl_cert, ssl_key))
+        app.run(port=443)