FM / exim_analytics

/usr/lib/rads/venv/lib/python3.13/site-packages/exim_analytics/

/usr/lib/rads/venv/lib/python3.13/site-packages/exim_analytics

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
api.py	2692	0644	edit dl rm
config.py	551	0644	edit dl rm
parser.py	3507	0644	edit dl rm
py.typed	0	0644	edit dl rm
scraper.py	4615	0644	edit dl rm
__init__.py	446	0644	edit dl rm

Edit: /usr/lib/rads/venv/lib/python3.13/site-packages/exim_analytics/parser.py (3507B)

import hashlib
import re
import json
import logging
import os

PATTERN_IN = re.compile(
    r"^(?P<date>\d{4}-\d{2}-\d{2}) "
    r"(?P<time>\d{2}:\d{2}:\d{2})\.\d+\s+"
    r"\[\d+\] (?P<msgid>\S+) <= (?P<env_sender>\S+).*?"
    r'T="(?P<subject>.*?)"\s+'
    r"from\s+<(?P<sender>[^>]+)>\s+"
    r"for\s+(?P<recipient>\S+)"
)

PATTERN_OUT = re.compile(
    r"^\d{4}-\d{2}-\d{2} "
    r"\d{2}:\d{2}:\d{2}\.\d+ "
    r"\[\d+\] (?P<msgid>\S+) => "
    r"(?P<recipient>\S+).*?"
    r"\bR=(?P<router>\S+)"
)

MSGID_MAP = {}

routers = [
    "remoteserver_route",
    "dkim_lookuphost",
]

def load_offset(json_file):
    try:
        with open(json_file, encoding="utf-8") as f:
            return json.load(f)
    except (FileNotFoundError, json.JSONDecodeError):
        return {}  # If the file doesn't exist or can't be decoded, start from 0

# Function to save the offset to a JSON file
def save_offset(json_file, offsets):
    with open(json_file, "w", encoding="utf-8") as f:
        json.dump(offsets, f, indent=4)

def parse_exim(OFFSET_FILE, EXIM_LOG, OUTPUT_LOG):
    offsets = load_offset(OFFSET_FILE)
    last_position = offsets.get(EXIM_LOG, 0)
    previous_first_line = offsets.get("exim_start_hash", 0)

if not os.path.exists(EXIM_LOG):
        return False
    
    # store the md5 of the first line
    with open(EXIM_LOG, encoding="utf-8", errors="replace") as f:
        current_first_line = f.readline().strip()
        current_first_line_hash = hashlib.md5(
            current_first_line.encode("utf-8")
        ).hexdigest()

# start at 0 if the first line md5s dont match or the log is smaller
    # than the offset
    if (
        previous_first_line != current_first_line_hash
        or last_position > os.path.getsize(EXIM_LOG)
    ):
        last_position = 0
        offsets["exim_start_hash"] = current_first_line_hash

# store <= lines
    with open(EXIM_LOG, encoding="utf-8", errors="replace") as f:
        f.seek(last_position)
        for line in f:
            m = PATTERN_IN.search(line)
            if m:
                msgid = m.group("msgid")
                MSGID_MAP[msgid] = {
                    "msgid": msgid,
                    "date": m.group("date"),
                    "time": m.group("time"),
                    "envelope_sender": m.group("env_sender"),
                    "subject": m.group("subject"),
                    "from": m.group("sender"),
                    "original_recipient": m.group("recipient"),
                }

# store => lines
    logging.debug("Parsing exim log: %s", EXIM_LOG)
    with (
        open(EXIM_LOG, encoding="utf-8", errors="replace") as f,
        open(OUTPUT_LOG, "a", encoding="utf-8") as out,
    ):
        f.seek(last_position)
        for line in f:
            m = PATTERN_OUT.search(line)
            if not m:
                continue

msgid = m.group("msgid")
            router = m.group("router")
            # identify outgoing message by checking for known routers
            if msgid in MSGID_MAP and router in routers:
                result = MSGID_MAP[msgid].copy()
                result["delivered_to"] = m.group("recipient")
                # result["router"] = router
                result["outbound_line"] = line.strip()
                out.write(json.dumps(result) + "\n")

# get end of file
        last_position = f.tell()
        offsets[EXIM_LOG] = last_position

# update offset json
    save_offset(OFFSET_FILE, offsets)
    return True

import hashlib import re import json import logging import os PATTERN_IN = re.compile( r"^(?P\d{4}-\d{2}-\d{2}) " r"(?P\d{2}:\d{2}:\d{2})\.\d+\s+" r"\[\d+\] (?P\S+) <= (?P\S+).*?" r'T="(?P.*?)"\s+' r"from\s+<(?P[^>]+)>\s+" r"for\s+(?P\S+)" ) PATTERN_OUT = re.compile( r"^\d{4}-\d{2}-\d{2} " r"\d{2}:\d{2}:\d{2}\.\d+ " r"\[\d+\] (?P\S+) => " r"(?P\S+).*?" r"\bR=(?P\S+)" ) MSGID_MAP = {} routers = [ "remoteserver_route", "dkim_lookuphost", ] def load_offset(json_file): try: with open(json_file, encoding="utf-8") as f: return json.load(f) except (FileNotFoundError, json.JSONDecodeError): return {} # If the file doesn't exist or can't be decoded, start from 0 # Function to save the offset to a JSON file def save_offset(json_file, offsets): with open(json_file, "w", encoding="utf-8") as f: json.dump(offsets, f, indent=4) def parse_exim(OFFSET_FILE, EXIM_LOG, OUTPUT_LOG): offsets = load_offset(OFFSET_FILE) last_position = offsets.get(EXIM_LOG, 0) previous_first_line = offsets.get("exim_start_hash", 0) if not os.path.exists(EXIM_LOG): return False # store the md5 of the first line with open(EXIM_LOG, encoding="utf-8", errors="replace") as f: current_first_line = f.readline().strip() current_first_line_hash = hashlib.md5( current_first_line.encode("utf-8") ).hexdigest() # start at 0 if the first line md5s dont match or the log is smaller # than the offset if ( previous_first_line != current_first_line_hash or last_position > os.path.getsize(EXIM_LOG) ): last_position = 0 offsets["exim_start_hash"] = current_first_line_hash # store <= lines with open(EXIM_LOG, encoding="utf-8", errors="replace") as f: f.seek(last_position) for line in f: m = PATTERN_IN.search(line) if m: msgid = m.group("msgid") MSGID_MAP[msgid] = { "msgid": msgid, "date": m.group("date"), "time": m.group("time"), "envelope_sender": m.group("env_sender"), "subject": m.group("subject"), "from": m.group("sender"), "original_recipient": m.group("recipient"), } # store => lines logging.debug("Parsing exim log: %s", EXIM_LOG) with ( open(EXIM_LOG, encoding="utf-8", errors="replace") as f, open(OUTPUT_LOG, "a", encoding="utf-8") as out, ): f.seek(last_position) for line in f: m = PATTERN_OUT.search(line) if not m: continue msgid = m.group("msgid") router = m.group("router") # identify outgoing message by checking for known routers if msgid in MSGID_MAP and router in routers: result = MSGID_MAP[msgid].copy() result["delivered_to"] = m.group("recipient") # result["router"] = router result["outbound_line"] = line.strip() out.write(json.dumps(result) + "\n") # get end of file last_position = f.tell() offsets[EXIM_LOG] = last_position # update offset json save_offset(OFFSET_FILE, offsets) return True