Compare commits

...

17 Commits

Author SHA1 Message Date
pixeebot[bot] a2621fc05b Verify SSL Certificates for Requests. 2024-05-02 03:43:55 +00:00
snorkell-ai[bot] 3ff9520114 snorkell_ai/config_54b37
main
2024-04-14 17:12:47 +00:00
Robert Bongart (MSc MSc MA) cf5808cf71 Merge pull request #1 from 2lambda123/dependabot/pip/assets/pip-33fa3f9920
Bump the pip group across 1 directory with 2 updates
2024-04-14 11:59:46 -05:00
Robert Bongart (MSc MSc MA) 052ce4cd9c Merge pull request #2 from 2lambda123/imgbot
[ImgBot] Optimize images
2024-04-14 11:59:41 -05:00
Robert Bongart (MSc MSc MA) 1eecd892f7 Merge pull request #9 from 2lambda123/ablesov/add_diagnostics_script
Added diagnostics script
2024-04-14 11:59:11 -05:00
Robert Bongart (MSc MSc MA) f9be1daccb Merge pull request #8 from 2lambda123/ablesov/fix_requirements
Update requirements.txt
2024-04-14 11:52:34 -05:00
Robert Bongart (MSc MSc MA) 6d0560c311 Merge pull request #6 from 2lambda123/ablesov/pwd_compare_fix
Update install.sh
2024-04-14 11:28:44 -05:00
Robert Bongart (MSc MSc MA) dfdbff5100 Merge pull request #3 from 2lambda123/ablesov/fix_field_name
Fixed field mapping @ zeekengine.py
2024-04-14 11:16:37 -05:00
dependabot[bot] e93a4c16de Bump the pip group across 1 directory with 2 updates
Bumps the pip group with 2 updates in the /assets directory: [flask](https://github.com/pallets/flask) and [pyjwt](https://github.com/jpadilla/pyjwt).


Updates `flask` from 1.1.2 to 2.2.5
- [Release notes](https://github.com/pallets/flask/releases)
- [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/flask/compare/1.1.2...2.2.5)

Updates `pyjwt` from 1.7.1 to 2.4.0
- [Release notes](https://github.com/jpadilla/pyjwt/releases)
- [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/jpadilla/pyjwt/compare/1.7.1...2.4.0)

---
updated-dependencies:
- dependency-name: flask
  dependency-type: direct:production
  dependency-group: pip
- dependency-name: pyjwt
  dependency-type: direct:production
  dependency-group: pip
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-04-14 16:08:55 +00:00
ImgBotApp bd2c567151 [ImgBot] Optimize images
*Total -- 5,245.48kb -> 4,078.69kb (22.24%)

/app/frontend/src/assets/icon.png -- 27.84kb -> 14.24kb (48.84%)
/assets/network-home.png -- 315.20kb -> 184.93kb (41.33%)
/app/backend/src/assets/network-home.png -- 314.77kb -> 185.00kb (41.23%)
/app/backend/src/assets/network.png -- 133.43kb -> 91.76kb (31.22%)
/assets/backend.png -- 775.25kb -> 540.26kb (30.31%)
/app/frontend/src/assets/logo.png -- 87.56kb -> 63.96kb (26.95%)
/assets/frontend.png -- 404.96kb -> 324.34kb (19.91%)
/app/frontend/src/assets/fonts/Roboto-Bold.svg -- 805.04kb -> 673.54kb (16.34%)
/app/backend/src/assets/fonts/Roboto-Bold.svg -- 805.04kb -> 673.54kb (16.34%)
/app/frontend/src/assets/fonts/Roboto-Regular.svg -- 784.41kb -> 660.24kb (15.83%)
/app/backend/src/assets/fonts/Roboto-Regular.svg -- 784.41kb -> 660.24kb (15.83%)
/app/backend/src/assets/github.png -- 4.17kb -> 3.52kb (15.51%)
/app/frontend/src/assets/loading.svg -- 1.21kb -> 1.05kb (13.12%)
/app/frontend/src/assets/icon_spinner.svg -- 0.95kb -> 0.85kb (9.9%)
/app/frontend/src/assets/icon_plug_usb.svg -- 0.29kb -> 0.27kb (4.42%)
/app/frontend/src/assets/icon_success.svg -- 0.37kb -> 0.35kb (3.71%)
/app/frontend/src/assets/icon_usb.svg -- 0.59kb -> 0.57kb (3.16%)

Signed-off-by: ImgBotApp <ImgBotHelp@gmail.com>
2024-04-14 16:08:12 +00:00
Eugeny Ablesov 4f989dcd94 Fixed field mapping @ zeekengine.py (#143)
Fixed field mapping @ zeekengine.py: resp_h is destination host, please refer to https://docs.zeek.org/en/current/log-formats.html
2023-08-14 15:14:42 +03:00
Eugeny Ablesov 4e4c6172cd Fixed field mapping @ zeekengine.py
Fixed field mapping @ zeekengine.py: resp_h is destination host, please refer to https://docs.zeek.org/en/current/log-formats.html
2023-08-14 15:14:00 +03:00
Eugeny Ablesov 9939dc3bf4 Added diagnostics script (#141) 2023-06-26 14:04:10 +03:00
Eugeny Ablesov c3f09469b9 Added diagnostics script 2023-06-26 14:01:58 +03:00
Eugeny Ablesov 5091308d0b Update install.sh (#140)
Fixed password comparison
2023-06-26 14:00:43 +03:00
Eugeny Ablesov 69512ba605 Update install.sh
Fixed password comparison
2023-06-26 14:00:12 +03:00
Eugeny Ablesov 0dafbf63a2 Update requirements.txt (#139)
Use only fixed versions for dependencies
2023-06-26 13:59:03 +03:00
23 changed files with 372 additions and 45425 deletions
@@ -0,0 +1,19 @@
# This workflow will improvise current file with AI genereated documentation and Create new PR
name: Snorkell.ai - Revolutionizing Documentation on GitHub
on:
push:
branches: ["main"]
workflow_dispatch:
jobs:
Documentation:
runs-on: ubuntu-latest
steps:
- name: Snorkell DocGen Client
uses: SingularityX-ai/snorkell-documentation-client@v1.0.0
with:
client_id: ${{ secrets.SNORKELL_CLIENT_ID }}
api_key: ${{ secrets.SNORKELL_API_KEY }}
branch_name: "main"
+1 -1
View File
@@ -270,7 +270,7 @@ class ZeekEngine(object):
if record is not None:
f = {"filename": record["filename"],
"ip_src": record["id.orig_h"],
"ip_dst": record["id.orig_p"],
"ip_dst": record["id.resp_h"],
"mime_type": record["mime_type"],
"sha1": record["sha1"]}
if f not in self.files:
File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 805 KiB

After

Width:  |  Height:  |  Size: 674 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 784 KiB

After

Width:  |  Height:  |  Size: 660 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.2 KiB

After

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 315 KiB

After

Width:  |  Height:  |  Size: 185 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 133 KiB

After

Width:  |  Height:  |  Size: 92 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 805 KiB

After

Width:  |  Height:  |  Size: 674 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 784 KiB

After

Width:  |  Height:  |  Size: 660 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 14 KiB

+1 -4
View File
@@ -1,4 +1 @@
<svg width="112" height="195" viewBox="0 0 112 195" fill="none" xmlns="http://www.w3.org/2000/svg">
<line x1="3.5" y1="3.5" x2="3.50001" y2="191.5" stroke="black" stroke-width="7" stroke-linecap="round" stroke-linejoin="round"/>
<rect x="7" width="105" height="195" fill="#F7F8F9"/>
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="112" height="195" fill="none" viewBox="0 0 112 195"><line x1="3.5" x2="3.5" y1="3.5" y2="191.5" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="7"/><rect width="105" height="195" x="7" fill="#F7F8F9"/></svg>

Before

Width:  |  Height:  |  Size: 294 B

After

Width:  |  Height:  |  Size: 281 B

+1 -16
View File
@@ -1,16 +1 @@
<svg version="1.1" id="loader-1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
width="40px" height="40px" viewBox="0 0 40 40" enable-background="new 0 0 40 40" xml:space="preserve">
<path opacity="0.2" fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946
s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634
c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z"/>
<path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0
C22.32,8.481,24.301,9.057,26.013,10.047z">
<animateTransform attributeType="xml"
attributeName="transform"
type="rotate"
from="0 20 20"
to="360 20 20"
dur="0.5s"
repeatCount="indefinite"/>
</path>
</svg>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="loader-1" width="40" height="40" x="0" y="0" enable-background="new 0 0 40 40" version="1.1" viewBox="0 0 40 40" xml:space="preserve"><path fill="#000" d="M20.201,5.169c-8.254,0-14.946,6.692-14.946,14.946c0,8.255,6.692,14.946,14.946,14.946 s14.946-6.691,14.946-14.946C35.146,11.861,28.455,5.169,20.201,5.169z M20.201,31.749c-6.425,0-11.634-5.208-11.634-11.634 c0-6.425,5.209-11.634,11.634-11.634c6.425,0,11.633,5.209,11.633,11.634C31.834,26.541,26.626,31.749,20.201,31.749z" opacity=".2"/><path fill="#f7f8f9" d="M26.013,10.047l1.654-2.866c-2.198-1.272-4.743-2.012-7.466-2.012h0v3.312h0 C22.32,8.481,24.301,9.057,26.013,10.047z"><animateTransform attributeName="transform" attributeType="xml" dur="0.5s" from="0 20 20" repeatCount="indefinite" to="360 20 20" type="rotate"/></path></svg>

Before

Width:  |  Height:  |  Size: 970 B

After

Width:  |  Height:  |  Size: 874 B

+1 -5
View File
@@ -1,5 +1 @@
<svg width="106" height="106" viewBox="0 0 106 106" fill="none" xmlns="http://www.w3.org/2000/svg">
<circle cx="53" cy="53" r="53" fill="#40D8A1"/>
<path d="M29 52.5L47.5 70.5" stroke="white" stroke-width="10" stroke-linecap="round" stroke-linejoin="round"/>
<line x1="79" y1="40.0711" x2="48.0711" y2="71" stroke="white" stroke-width="10" stroke-linecap="round"/>
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="106" fill="none" viewBox="0 0 106 106"><circle cx="53" cy="53" r="53" fill="#40D8A1"/><path stroke="#fff" stroke-linecap="round" stroke-linejoin="round" stroke-width="10" d="M29 52.5L47.5 70.5"/><line x1="79" x2="48.071" y1="40.071" y2="71" stroke="#fff" stroke-linecap="round" stroke-width="10"/></svg>

Before

Width:  |  Height:  |  Size: 377 B

After

Width:  |  Height:  |  Size: 363 B

+1 -6
View File
@@ -1,6 +1 @@
<svg width="548" height="199" viewBox="0 0 548 199" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect x="403" y="27" width="142" height="145" rx="8" fill="white" stroke="black" stroke-width="6" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z" fill="black"/>
<rect x="477" y="55" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
<rect x="477" y="117" width="26" height="26" fill="white" stroke="black" stroke-width="6"/>
</svg>
<svg xmlns="http://www.w3.org/2000/svg" width="548" height="199" fill="none" viewBox="0 0 548 199"><rect width="142" height="145" x="403" y="27" fill="#fff" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="6" rx="8"/><path fill="#000" d="M0 30C0 13.4315 13.4315 0 30 0H428C432.418 0 436 3.58172 436 8V191C436 195.418 432.418 199 428 199H30C13.4315 199 0 185.569 0 169V30Z"/><rect width="26" height="26" x="477" y="55" fill="#fff" stroke="#000" stroke-width="6"/><rect width="26" height="26" x="477" y="117" fill="#fff" stroke="#000" stroke-width="6"/></svg>

Before

Width:  |  Height:  |  Size: 602 B

After

Width:  |  Height:  |  Size: 583 B

+1 -11
View File
@@ -1,11 +1 @@
<?xml version="1.0" encoding="utf-8"?>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin: auto; background: none; display: block; shape-rendering: auto;" width="200px" height="200px" viewBox="0 0 100 100" preserveAspectRatio="xMidYMid">
<circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2">
<animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
<animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline" begin="-0.7462686567164178s"></animate>
</circle>
<circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2">
<animate attributeName="r" repeatCount="indefinite" dur="1.4925373134328357s" values="0;30" keyTimes="0;1" keySplines="0 0.2 0.8 1" calcMode="spline"></animate>
<animate attributeName="opacity" repeatCount="indefinite" dur="1.4925373134328357s" values="1;0" keyTimes="0;1" keySplines="0.2 0 0.8 1" calcMode="spline"></animate>
</circle>
<!-- [ldio] generated by https://loading.io/ --></svg>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="margin:auto;background:0 0;display:block;shape-rendering:auto" width="200" height="200" preserveAspectRatio="xMidYMid" viewBox="0 0 100 100"><circle cx="50" cy="50" r="0" fill="none" stroke="#f3f3f3" stroke-width="2"><animate attributeName="r" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" begin="-0.7462686567164178s" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle><circle cx="50" cy="50" r="0" fill="none" stroke="#d8dddf" stroke-width="2"><animate attributeName="r" calcMode="spline" dur="1.4925373134328357s" keySplines="0 0.2 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="0;30"/><animate attributeName="opacity" calcMode="spline" dur="1.4925373134328357s" keySplines="0.2 0 0.8 1" keyTimes="0;1" repeatCount="indefinite" values="1;0"/></circle></svg>

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 88 KiB

After

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 775 KiB

After

Width:  |  Height:  |  Size: 540 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 405 KiB

After

Width:  |  Height:  |  Size: 324 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 315 KiB

After

Width:  |  Height:  |  Size: 185 KiB

+2 -2
View File
@@ -2,9 +2,9 @@ pymisp==2.4.165.1
sqlalchemy==1.4.48
ipwhois==1.2.0
netaddr==0.8.0
flask==1.1.2
flask==2.2.5
flask_httpauth==4.8.0
pyjwt==1.7.1
pyjwt==2.4.0
psutil==5.8.0
pydig==0.4.0
pyudev==0.24.0
+1 -1
View File
@@ -69,7 +69,7 @@ set_credentials() {
read -s password2
echo ""
if [ $password1 = $password2 ]; then
if [ "$password1" == "$password2" ]; then
password=$(echo -n "$password1" | sha256sum | cut -d" " -f1)
sed -i "s/userlogin/$login/g" /usr/share/tinycheck/config.yaml
sed -i "s/userpassword/$password/g" /usr/share/tinycheck/config.yaml
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/python
import os
import subprocess
import platform
import socket
import pkg_resources
import psutil
__author__ = 'Eugeny N Ablesov'
__version__ = '1.0.17'
def collect_accounts_info():
""" This call collects generic information about
user accounts presented on system running TinyCheck.
No personal information collected or provided by this call.
"""
accs = { }
users = psutil.users()
for user in users:
accs[user.name + '@' + user.host] = {
'started': user.started,
'term': user.terminal
}
alt_user = os.getenv('SUDO_USER', os.getenv('USER'))
usr = 'root' if os.path.expanduser('~') == '/root' else alt_user
pid = psutil.Process().pid
term = psutil.Process().terminal() if 'Linux' in platform.system() else 'win'
accs[usr + '@' + term] = { 'pid': pid }
return accs
def collect_os_info():
""" This call collects generic information about
operating system running TinyCheck.
No personal information collected or provided by this call.
"""
os_info = { }
os_info['system'] = platform.system()
os_info['release'] = platform.release()
os_info['version'] = platform.version()
os_info['platform'] = platform.platform(aliased=True)
if 'Windows' in os_info['system']:
os_info['dist'] = platform.win32_ver()
if 'Linux' in os_info['system']:
os_info['dist'] = platform.libc_ver()
return os_info
def collect_hardware_info():
""" This call collects information about hardware running TinyCheck.
No personal information collected or provided by this call.
"""
hw_info = { }
hw_info['arch'] = platform.architecture()
hw_info['machine'] = platform.machine()
hw_info['cpus'] = psutil.cpu_count(logical=False)
hw_info['cores'] = psutil.cpu_count()
hw_info['load'] = psutil.getloadavg()
disk_info = psutil.disk_usage('/')
hw_info['disk'] = {
'total': disk_info.total,
'used': disk_info.used,
'free': disk_info.free
}
return hw_info
def collect_network_info():
""" This call collects information about
network configuration and state running TinyCheck.
No personal information collected or provided by this call.
"""
net_info = { }
net_info['namei'] = socket.if_nameindex()
addrs = psutil.net_if_addrs()
state = psutil.net_io_counters(pernic=True)
for interface in addrs.keys():
net_info[interface] = { }
int_info = state[interface]
props = [p for p in dir(int_info)
if not p.startswith("_")
and not p == "index"
and not p == "count"]
for prop in props:
net_info[interface][prop] = getattr(int_info, prop)
return net_info
def collect_dependency_info(package_list):
""" This call collects information about
python packages required to run TinyCheck.
No personal information collected or provided by this call.
"""
dependencies = { }
installed_packages = list(pkg_resources.working_set)
installed_packages_list = sorted(["%s==%s"
% (installed.key, installed.version)
for installed in installed_packages])
for pkg in installed_packages_list:
[package_name, package_version] = pkg.split('==')
if package_name in package_list:
dependencies[package_name] = package_version
return dependencies
def collect_db_tables_records_count(db_path, tables):
result = { }
for table in tables:
query = 'SELECT COUNT(*) FROM %s' % (table)
sqlite_call = subprocess.Popen(['sqlite3', db_path, query], stdout = subprocess.PIPE)
stout, sterr = sqlite_call.communicate()
val = stout.decode("utf-8")
recs = int(val) if val else 0
result[table] = recs
return result
def collect_internal_state(db_path, tables, to_check):
""" This call collects information about
installed TinyCheck instance and its internal state.
No personal information collected or provided by this call.
"""
state_ = { }
available = os.path.isfile(db_path)
dbsize = 0
state_['db'] = {
'available': available,
'size': dbsize
}
state_['db']['records'] = { }
if available:
state_['db']['size'] = os.stat(db_path).st_size
state_['db']['records'] = collect_db_tables_records_count(db_path, tables)
services_ = { }
for alias in to_check:
status = subprocess.call(['systemctl', 'is-active', '--quiet', '%s' % (to_check[alias])])
state = ''
if status != 0:
sysctl_call = subprocess.Popen(
["systemctl", "status", "%s" % (to_check[alias]),
r"|",
"grep",
r"''"],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
stout, sterr = sysctl_call.communicate()
state = stout.decode("utf-8")
errs = sterr.decode("utf-8")
if "could not be found" in errs:
state = 'Service not found'
services_[alias] = {
'running': status == 0,
'status': status,
'state': state
}
state_['svc'] = services_
return state_
def main():
print("TinyCheck diagnostics script.\nVersion: %s" % (__version__))
print("")
db_path = '/usr/share/tinycheck/tinycheck.sqlite3'
tables = ['iocs', 'whitelist', 'misp']
services = { }
services['frontend'] = 'tinycheck-frontend.service'
services['backend'] = 'tinycheck-backend.service'
services['kiosk'] = 'tinycheck-kiosk.service'
services['watchers'] = 'tinycheck-watchers.service'
deps = [
'pymisp', 'sqlalchemy', 'ipwhois',
'netaddr', 'flask', 'flask_httpauth',
'pyjwt', 'psutil', 'pydig', 'pyudev',
'pyyaml', 'wifi', 'qrcode', 'netifaces',
'weasyprint', 'python-whois', 'six' ]
diagnostics = { }
diagnostics['acc'] = collect_accounts_info()
diagnostics['os'] = collect_os_info()
diagnostics['hw'] = collect_hardware_info()
diagnostics['net'] = collect_network_info()
diagnostics['deps'] = collect_dependency_info(deps)
diagnostics['state'] = collect_internal_state(db_path, tables, services)
report = { 'diagnostics': diagnostics }
print(report)
print("")
if __name__ == '__main__':
main()
+149 -149
View File
@@ -1,149 +1,149 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from app.utils import read_config
from app.classes.iocs import IOCs
from app.classes.whitelist import WhiteList
from app.classes.misp import MISP
import requests
import json
import urllib3
import time
from multiprocessing import Process
"""
This file is parsing the watchers present
in the configuration file. This in order to get
automatically new iocs / elements from remote
sources without user interaction.
"""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def watch_iocs():
"""
Retrieve IOCs from the remote URLs defined in config/watchers.
For each IOC, add it to the DB.
"""
# Retrieve the URLs from the configuration
urls = read_config(("watchers", "iocs"))
watchers = [{"url": url, "status": False} for url in urls]
while True:
for w in watchers:
if w["status"] == False:
iocs = IOCs()
iocs_list = []
to_delete = []
try:
res = requests.get(w["url"], verify=False)
if res.status_code == 200:
content = json.loads(res.content)
iocs_list = content["iocs"] if "iocs" in content else []
to_delete = content["to_delete"] if "to_delete" in content else []
else:
w["status"] = False
except:
w["status"] = False
for ioc in iocs_list:
try:
iocs.add(ioc["type"], ioc["tag"],
ioc["tlp"], ioc["value"], "watcher")
w["status"] = True
except:
continue
for ioc in to_delete:
try:
iocs.delete_by_value(ioc["value"])
w["status"] = True
except:
continue
# If at least one URL haven't be parsed, let's retry in 1min.
if False in [w["status"] for w in watchers]:
time.sleep(60)
else:
break
def watch_whitelists():
"""
Retrieve whitelist elements from the remote URLs
defined in config/watchers. For each (new ?) element,
add it to the DB.
"""
urls = read_config(("watchers", "whitelists"))
watchers = [{"url": url, "status": False} for url in urls]
while True:
for w in watchers:
if w["status"] == False:
whitelist = WhiteList()
elements = []
to_delete = []
try:
res = requests.get(w["url"], verify=False)
if res.status_code == 200:
content = json.loads(res.content)
elements = content["elements"] if "elements" in content else []
to_delete = content["to_delete"] if "to_delete" in content else []
else:
w["status"] = False
except:
w["status"] = False
for elem in elements:
try:
whitelist.add(elem["type"], elem["element"], "watcher")
w["status"] = True
except:
continue
for elem in to_delete:
try:
whitelist.delete_by_value(elem["element"])
w["status"] = True
except:
continue
if False in [w["status"] for w in watchers]:
time.sleep(60)
else:
break
def watch_misp():
"""
Retrieve IOCs from misp instances. Each new element is
tested and then added to the database.
"""
iocs, misp = IOCs(), MISP()
instances = [i for i in misp.get_instances()]
while instances:
for i, ist in enumerate(instances):
status = misp.test_instance(ist["url"],
ist["apikey"],
ist["verifycert"])
if status:
for ioc in misp.get_iocs(ist["id"]):
iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
ioc["value"], "misp-{}".format(ist["id"]))
misp.update_sync(ist["id"])
instances.pop(i)
if instances: time.sleep(60)
p1 = Process(target=watch_iocs)
p2 = Process(target=watch_whitelists)
p3 = Process(target=watch_misp)
p1.start()
p2.start()
p3.start()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from app.utils import read_config
from app.classes.iocs import IOCs
from app.classes.whitelist import WhiteList
from app.classes.misp import MISP
import requests
import json
import urllib3
import time
from multiprocessing import Process
"""
This file is parsing the watchers present
in the configuration file. This in order to get
automatically new iocs / elements from remote
sources without user interaction.
"""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def watch_iocs():
"""
Retrieve IOCs from the remote URLs defined in config/watchers.
For each IOC, add it to the DB.
"""
# Retrieve the URLs from the configuration
urls = read_config(("watchers", "iocs"))
watchers = [{"url": url, "status": False} for url in urls]
while True:
for w in watchers:
if w["status"] == False:
iocs = IOCs()
iocs_list = []
to_delete = []
try:
res = requests.get(w["url"], verify=True)
if res.status_code == 200:
content = json.loads(res.content)
iocs_list = content["iocs"] if "iocs" in content else []
to_delete = content["to_delete"] if "to_delete" in content else []
else:
w["status"] = False
except:
w["status"] = False
for ioc in iocs_list:
try:
iocs.add(ioc["type"], ioc["tag"],
ioc["tlp"], ioc["value"], "watcher")
w["status"] = True
except:
continue
for ioc in to_delete:
try:
iocs.delete_by_value(ioc["value"])
w["status"] = True
except:
continue
# If at least one URL haven't be parsed, let's retry in 1min.
if False in [w["status"] for w in watchers]:
time.sleep(60)
else:
break
def watch_whitelists():
"""
Retrieve whitelist elements from the remote URLs
defined in config/watchers. For each (new ?) element,
add it to the DB.
"""
urls = read_config(("watchers", "whitelists"))
watchers = [{"url": url, "status": False} for url in urls]
while True:
for w in watchers:
if w["status"] == False:
whitelist = WhiteList()
elements = []
to_delete = []
try:
res = requests.get(w["url"], verify=True)
if res.status_code == 200:
content = json.loads(res.content)
elements = content["elements"] if "elements" in content else []
to_delete = content["to_delete"] if "to_delete" in content else []
else:
w["status"] = False
except:
w["status"] = False
for elem in elements:
try:
whitelist.add(elem["type"], elem["element"], "watcher")
w["status"] = True
except:
continue
for elem in to_delete:
try:
whitelist.delete_by_value(elem["element"])
w["status"] = True
except:
continue
if False in [w["status"] for w in watchers]:
time.sleep(60)
else:
break
def watch_misp():
"""
Retrieve IOCs from misp instances. Each new element is
tested and then added to the database.
"""
iocs, misp = IOCs(), MISP()
instances = [i for i in misp.get_instances()]
while instances:
for i, ist in enumerate(instances):
status = misp.test_instance(ist["url"],
ist["apikey"],
ist["verifycert"])
if status:
for ioc in misp.get_iocs(ist["id"]):
iocs.add(ioc["type"], ioc["tag"], ioc["tlp"],
ioc["value"], "misp-{}".format(ist["id"]))
misp.update_sync(ist["id"])
instances.pop(i)
if instances: time.sleep(60)
p1 = Process(target=watch_iocs)
p2 = Process(target=watch_whitelists)
p3 = Process(target=watch_misp)
p1.start()
p2.start()
p3.start()