13 Commits

Author SHA1 Message Date
7879dd3aac Limit readline() 2024-04-16 04:51:38 +00:00
3ff9520114 snorkell_ai/config_54b37
main
2024-04-14 17:12:47 +00:00
cf5808cf71 Merge pull request #1 from 2lambda123/dependabot/pip/assets/pip-33fa3f9920
Bump the pip group across 1 directory with 2 updates
2024-04-14 11:59:46 -05:00
052ce4cd9c Merge pull request #2 from 2lambda123/imgbot
[ImgBot] Optimize images
2024-04-14 11:59:41 -05:00
1eecd892f7 Merge pull request #9 from 2lambda123/ablesov/add_diagnostics_script
Added diagnostics script
2024-04-14 11:59:11 -05:00
f9be1daccb Merge pull request #8 from 2lambda123/ablesov/fix_requirements
Update requirements.txt
2024-04-14 11:52:34 -05:00
6d0560c311 Merge pull request #6 from 2lambda123/ablesov/pwd_compare_fix
Update install.sh
2024-04-14 11:28:44 -05:00
dfdbff5100 Merge pull request #3 from 2lambda123/ablesov/fix_field_name
Fixed field mapping @ zeekengine.py
2024-04-14 11:16:37 -05:00
e93a4c16de Bump the pip group across 1 directory with 2 updates
Bumps the pip group with 2 updates in the /assets directory: [flask](https://github.com/pallets/flask) and [pyjwt](https://github.com/jpadilla/pyjwt).


Updates `flask` from 1.1.2 to 2.2.5
- [Release notes](https://github.com/pallets/flask/releases)
- [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/flask/compare/1.1.2...2.2.5)

Updates `pyjwt` from 1.7.1 to 2.4.0
- [Release notes](https://github.com/jpadilla/pyjwt/releases)
- [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/jpadilla/pyjwt/compare/1.7.1...2.4.0)

---
updated-dependencies:
- dependency-name: flask
  dependency-type: direct:production
  dependency-group: pip
- dependency-name: pyjwt
  dependency-type: direct:production
  dependency-group: pip
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-04-14 16:08:55 +00:00
4e4c6172cd Fixed field mapping @ zeekengine.py
Fixed field mapping @ zeekengine.py: resp_h is destination host, please refer to https://docs.zeek.org/en/current/log-formats.html
2023-08-14 15:14:00 +03:00
c3f09469b9 Added diagnostics script 2023-06-26 14:01:58 +03:00
69512ba605 Update install.sh
Fixed password comparison
2023-06-26 14:00:12 +03:00
bd4ab27c21 Update requirements.txt
Use only fixed versions for dependencies
2023-06-26 13:57:42 +03:00
3 changed files with 199 additions and 180 deletions

View File

@ -0,0 +1,19 @@
# This workflow will improvise current file with AI genereated documentation and Create new PR
name: Snorkell.ai - Revolutionizing Documentation on GitHub
on:
push:
branches: ["main"]
workflow_dispatch:
jobs:
Documentation:
runs-on: ubuntu-latest
steps:
- name: Snorkell DocGen Client
uses: SingularityX-ai/snorkell-documentation-client@v1.0.0
with:
client_id: ${{ secrets.SNORKELL_CLIENT_ID }}
api_key: ${{ secrets.SNORKELL_API_KEY }}
branch_name: "main"

View File

@ -1,178 +1,178 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from json import loads, dumps
from collections import OrderedDict
from datetime import datetime
from traceback import print_exc
# Taken from https://github.com/dgunter/ParseZeekLogs <3
class ParseZeekLogs(object):
"""
Class that parses Zeek logs and allows log data to be output in CSV or json format.
Attributes: filepath: Path of Zeek log file to read
"""
def __init__(self, filepath, batchsize=500, fields=None, output_format=None, ignore_keys=[], meta={}, safe_headers=False):
self.fd = open(filepath, "r")
self.options = OrderedDict()
self.firstRun = True
self.filtered_fields = fields
self.batchsize = batchsize
self.output_format = output_format
self.ignore_keys = ignore_keys
self.meta = meta
self.safe_headers = safe_headers
# Convert ' to " in meta string
meta = loads(dumps(meta).replace("'", '"'))
# Read the header option lines
l = self.fd.readline().strip()
while l.strip().startswith("#"):
# Parse the options out
if l.startswith("#separator"):
key = str(l[1:].split(" ")[0])
value = str.encode(l[1:].split(
" ")[1].strip()).decode('unicode_escape')
self.options[key] = value
elif l.startswith("#"):
key = str(l[1:].split(self.options.get('separator'))[0])
value = l[1:].split(self.options.get('separator'))[1:]
self.options[key] = value
# Read the next line
l = self.fd.readline().strip()
self.firstLine = l
# Save mapping of fields to values:
self.fields = self.options.get('fields')
self.types = self.options.get('types')
self.data_types = {}
for i, val in enumerate(self.fields):
# Convert field names if safe_headers is enabled
if self.safe_headers is True:
self.fields[i] = self.fields[i].replace(".", "_")
# Match types with each other
self.data_types[self.fields[i]] = self.types[i]
def __del__(self):
self.fd.close()
def __iter__(self):
return self
def __next__(self):
retVal = ""
if self.firstRun is True:
retVal = self.firstLine
self.firstRun = False
else:
retVal = self.fd.readline().strip()
# If an empty string is returned, readline is done reading
if retVal == "" or retVal is None:
raise StopIteration
# Split out the data we are going to return
retVal = retVal.split(self.options.get('separator'))
record = None
# Make sure we aren't dealing with a comment line
if len(retVal) > 0 and not str(retVal[0]).strip().startswith("#") \
and len(retVal) is len(self.options.get("fields")):
record = OrderedDict()
# Prepare fields for conversion
for x in range(0, len(retVal)):
if self.safe_headers is True:
converted_field_name = self.options.get(
"fields")[x].replace(".", "_")
else:
converted_field_name = self.options.get("fields")[x]
if self.filtered_fields is None or converted_field_name in self.filtered_fields:
# Translate - to "" to fix a conversation error
if retVal[x] == "-":
retVal[x] = ""
# Save the record field if the field isn't filtered out
record[converted_field_name] = retVal[x]
# Convert values to the appropriate record type
record = self.convert_values(
record, self.ignore_keys, self.data_types)
if record is not None and self.output_format == "json":
# Output will be json
# Add metadata to json
for k, v in self.meta.items():
record[k] = v
retVal = record
elif record is not None and self.output_format == "csv":
retVal = ""
# Add escaping to csv format
for k, v in record.items():
# Add escaping to string values
if isinstance(v, str):
retVal += str("\"" + str(v).strip() + "\"" + ",")
else:
retVal += str(str(v).strip() + ",")
# Remove the trailing comma
retVal = retVal[:-1]
else:
retVal = None
return retVal
def convert_values(self, data, ignore_keys=[], data_types={}):
keys_to_delete = []
for k, v in data.items():
# print("evaluating k: " + str(k) + " v: " + str(v))
if isinstance(v, dict):
data[k] = self.convert_values(v)
else:
if data_types.get(k) is not None:
if (data_types.get(k) == "port" or data_types.get(k) == "count"):
if v != "":
data[k] = int(v)
else:
keys_to_delete.append(k)
elif (data_types.get(k) == "double" or data_types.get(k) == "interval"):
if v != "":
data[k] = float(v)
else:
keys_to_delete.append(k)
elif data_types.get(k) == "bool":
data[k] = bool(v)
else:
data[k] = v
for k in keys_to_delete:
del data[k]
return data
def get_fields(self):
"""Returns all fields present in the log file
Returns:
A python list containing all field names in the log file
"""
field_names = ""
if self.output_format == "csv":
for i, v in enumerate(self.fields):
if self.filtered_fields is None or v in self.filtered_fields:
field_names += str(v) + ","
# Remove the trailing comma
field_names = field_names[:-1].strip()
else:
field_names = []
for i, v in enumerate(self.fields):
if self.filtered_fields is None or v in self.filtered_fields:
field_names.append(v)
return field_names
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from json import loads, dumps
from collections import OrderedDict
from datetime import datetime
from traceback import print_exc
# Taken from https://github.com/dgunter/ParseZeekLogs <3
class ParseZeekLogs(object):
"""
Class that parses Zeek logs and allows log data to be output in CSV or json format.
Attributes: filepath: Path of Zeek log file to read
"""
def __init__(self, filepath, batchsize=500, fields=None, output_format=None, ignore_keys=[], meta={}, safe_headers=False):
self.fd = open(filepath, "r")
self.options = OrderedDict()
self.firstRun = True
self.filtered_fields = fields
self.batchsize = batchsize
self.output_format = output_format
self.ignore_keys = ignore_keys
self.meta = meta
self.safe_headers = safe_headers
# Convert ' to " in meta string
meta = loads(dumps(meta).replace("'", '"'))
# Read the header option lines
l = self.fd.readline(5_000_000).strip()
while l.strip().startswith("#"):
# Parse the options out
if l.startswith("#separator"):
key = str(l[1:].split(" ")[0])
value = str.encode(l[1:].split(
" ")[1].strip()).decode('unicode_escape')
self.options[key] = value
elif l.startswith("#"):
key = str(l[1:].split(self.options.get('separator'))[0])
value = l[1:].split(self.options.get('separator'))[1:]
self.options[key] = value
# Read the next line
l = self.fd.readline(5_000_000).strip()
self.firstLine = l
# Save mapping of fields to values:
self.fields = self.options.get('fields')
self.types = self.options.get('types')
self.data_types = {}
for i, val in enumerate(self.fields):
# Convert field names if safe_headers is enabled
if self.safe_headers is True:
self.fields[i] = self.fields[i].replace(".", "_")
# Match types with each other
self.data_types[self.fields[i]] = self.types[i]
def __del__(self):
self.fd.close()
def __iter__(self):
return self
def __next__(self):
retVal = ""
if self.firstRun is True:
retVal = self.firstLine
self.firstRun = False
else:
retVal = self.fd.readline().strip()
# If an empty string is returned, readline is done reading
if retVal == "" or retVal is None:
raise StopIteration
# Split out the data we are going to return
retVal = retVal.split(self.options.get('separator'))
record = None
# Make sure we aren't dealing with a comment line
if len(retVal) > 0 and not str(retVal[0]).strip().startswith("#") \
and len(retVal) is len(self.options.get("fields")):
record = OrderedDict()
# Prepare fields for conversion
for x in range(0, len(retVal)):
if self.safe_headers is True:
converted_field_name = self.options.get(
"fields")[x].replace(".", "_")
else:
converted_field_name = self.options.get("fields")[x]
if self.filtered_fields is None or converted_field_name in self.filtered_fields:
# Translate - to "" to fix a conversation error
if retVal[x] == "-":
retVal[x] = ""
# Save the record field if the field isn't filtered out
record[converted_field_name] = retVal[x]
# Convert values to the appropriate record type
record = self.convert_values(
record, self.ignore_keys, self.data_types)
if record is not None and self.output_format == "json":
# Output will be json
# Add metadata to json
for k, v in self.meta.items():
record[k] = v
retVal = record
elif record is not None and self.output_format == "csv":
retVal = ""
# Add escaping to csv format
for k, v in record.items():
# Add escaping to string values
if isinstance(v, str):
retVal += str("\"" + str(v).strip() + "\"" + ",")
else:
retVal += str(str(v).strip() + ",")
# Remove the trailing comma
retVal = retVal[:-1]
else:
retVal = None
return retVal
def convert_values(self, data, ignore_keys=[], data_types={}):
keys_to_delete = []
for k, v in data.items():
# print("evaluating k: " + str(k) + " v: " + str(v))
if isinstance(v, dict):
data[k] = self.convert_values(v)
else:
if data_types.get(k) is not None:
if (data_types.get(k) == "port" or data_types.get(k) == "count"):
if v != "":
data[k] = int(v)
else:
keys_to_delete.append(k)
elif (data_types.get(k) == "double" or data_types.get(k) == "interval"):
if v != "":
data[k] = float(v)
else:
keys_to_delete.append(k)
elif data_types.get(k) == "bool":
data[k] = bool(v)
else:
data[k] = v
for k in keys_to_delete:
del data[k]
return data
def get_fields(self):
"""Returns all fields present in the log file
Returns:
A python list containing all field names in the log file
"""
field_names = ""
if self.output_format == "csv":
for i, v in enumerate(self.fields):
if self.filtered_fields is None or v in self.filtered_fields:
field_names += str(v) + ","
# Remove the trailing comma
field_names = field_names[:-1].strip()
else:
field_names = []
for i, v in enumerate(self.fields):
if self.filtered_fields is None or v in self.filtered_fields:
field_names.append(v)
return field_names

View File

@ -2,9 +2,9 @@ pymisp==2.4.165.1
sqlalchemy==1.4.48
ipwhois==1.2.0
netaddr==0.8.0
flask==1.1.2
flask==2.2.5
flask_httpauth==4.8.0
pyjwt==1.7.1
pyjwt==2.4.0
psutil==5.8.0
pydig==0.4.0
pyudev==0.24.0