I've written a little piece of code to filter Firewall-Connection-Events by a certain group of IPs. But the code can't keep up since the input is fairly huge. I am looking for ways to make this code more efficient in order to handle the Firewall-Connection-Events.
[READ_CLIENTS] The code starts with an empty list (client_IPs). The list is getting filled with unique IPs (read_clients), if the size of the file changes "read_clients" is called again to alter the list.
[READ_EVENTS] To get the events, I've used While True to loop over the event-File and return the events with yield - if there is no new input, then sleep for 0.1 seconds.
[PROCESS_AND_FILTER] After this I loop over the generator object to compare each event to each Unique-IP and seperate the result in two files.
# -*- coding: utf-8 -*-
from IPy import IP
import os
import time
# Create String-Array to compare
path_unique_ips = '/var/log/unique.log'
# Sophos-UTM Packet-Accepted-Event | Connections to Customer-Net
path_sophos_to_customer = '/var/log/packet-accepted-sophos.log'
# match logs
match_log = '/var/log/matched.log'
no_match_log = '/var/log/not_matched.log'
# IP-Filter-Array
client_IPs = []
#get file size of unique ips file
size_ip_file = os.stat(path_unique_ips).st_size
def read_clients(path_unique_ips):
client_IPs_file = open(path_unique_ips, "r")
if client_IPs_file.mode == 'r':
# read all line from client_IPs_file
new_client_IPs = client_IPs_file.readlines()
#check for new clients and fill array
for new_client_IP in new_client_IPs:
if new_client_IP not in client_IPs:
client_IPs.append(IP(new_client_IP).strNormal())
def read_events(path_sophos_to_customer):
connection_event_to_customer = open(path_sophos_to_customer, 'r')
connection_event_to_customer.seek(0, 2)
if connection_event_to_customer.mode == 'r':
while True:
new_event = connection_event_to_customer.readline()
if not new_event:
time.sleep(0.1)
continue
yield new_event
#file size of unique IP File changed, re-run the function read_clients
if size_ip_file is not os.stat(path_unique_ips).st_size:
read_clients(path_unique_ips)
def process_and_filter(my_events):
#get events in generator-object from function read_events
# loop over generator-object, filled with events
for new_event in my_events:
print(new_event)
# loop over event with all ips
for client in client_IPs:
# if client-ip found in event write to match.log and break loop to go for next event
if client in new_event:
with open(match_log, 'a+') as matched:
matched.write(new_event)
break
# if ip wasn't in event write event to no_match.log
else:
with open(no_match_log, 'a+') as no_match:
no_match.write(new_event)
if __name__ == '__main__':
read_clients(path_unique_ips)
new_events=read_events(path_sophos_to_customer)
process_and_filter(new_events)
Log_Event_Example :
Jan 18 14:14:14 17.17.17.17 2019: 01:18-14:14:14 firewall-1 ulogd[5974]: id="2002" severity="info" sys="SecureNet" sub="packetfilter" name="Packet accepted" action="accept" fwrule="653" initf="eth1" outitf="eth0" srcmac="aa:bb:cc:dd:ee:ff" dstmac="00:11:22:33:44:55" srcip="10.10.10.10" dstip="10.10.10.11" proto="6" length="52" tos="0x00" prec="0x00" ttl="127" srcport="58589" dstport="22" tcpflags="ACK"