diff --git a/.env.example b/.env.example index 100e376..3a79fcd 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,15 @@ JOURNAL_UNIT=mailserver # outside these ranges are flagged in the report as noteworthy. TRUSTED_LOGIN_NETWORKS=10.0.0.0/24 +# --- Multi-day repeat-offender tracking --- +# Requires the /data mount added to the systemd unit (bind-mounted, read-write) +# so this state survives across container restarts/daily runs. +STATE_FILE=/data/offender_state.json +# An IP gets called out as a repeat offender once seen on this many distinct days +REPEAT_OFFENDER_THRESHOLD_DAYS=3 +# IPs not seen again within this many days are dropped from tracking +OFFENDER_RETENTION_DAYS=30 + # --- Execution Options --- # Set to 'true' to see detailed background processes, 'false' for clean logs DEBUG=true diff --git a/daily_summary.py b/daily_summary.py index 9554d52..22d0810 100644 --- a/daily_summary.py +++ b/daily_summary.py @@ -1,10 +1,12 @@ import os import re +import json import ipaddress import subprocess import requests import smtplib from email.message import EmailMessage +from datetime import date, timedelta import time import schedule from collections import Counter, defaultdict @@ -37,6 +39,13 @@ TRUSTED_LOGIN_NETWORKS = [ n.strip() for n in os.getenv("TRUSTED_LOGIN_NETWORKS", "10.0.0.0/24").split(",") if n.strip() ] +# --- Multi-day repeat-offender tracking (persisted on the /data mount) --- +STATE_FILE = os.getenv("STATE_FILE", "/data/offender_state.json") +# An IP is called out as a "repeat offender" once it's shown up on this many distinct days +REPEAT_OFFENDER_THRESHOLD_DAYS = int(os.getenv("REPEAT_OFFENDER_THRESHOLD_DAYS", 3)) +# IPs not seen again after this many days are dropped from state, so the file doesn't grow forever +OFFENDER_RETENTION_DAYS = int(os.getenv("OFFENDER_RETENTION_DAYS", 30)) + # --- EXECUTION TOGGLES --- DEBUG = os.getenv("DEBUG", "false").lower() in ("true", "1", "yes") RUN_NOW = os.getenv("RUN_NOW", "false").lower() in ("true", "1", "yes") @@ -78,9 +87,9 @@ def _is_trusted(ip): # --- Regexes for structured extraction (mail server specific) --- CONNECT_RE = re.compile(r'postscreen\[\d+\]: CONNECT from \[([\d.:a-fA-F]+)\]') -REJECT_RE = re.compile(r'postscreen\[\d+\]: NOQUEUE: reject: RCPT from \[[^\]]+\]:\d+: (\d{3} \S+ [^;]+)') +REJECT_RE = re.compile(r'postscreen\[\d+\]: NOQUEUE: reject: RCPT from \[([\d.:a-fA-F]+)\]:\d+: (\d{3} \S+ [^;]+)') DNSBL_RE = re.compile(r'dnsblog\[\d+\]: addr') -RELAY_DENIED_RE = re.compile(r'reject: RCPT from \S+: 554 [\d.]+ <[^>]+>: Relay access denied.*from=<([^>]+)>') +RELAY_DENIED_RE = re.compile(r'reject: RCPT from \S+\[([\d.:a-fA-F]+)\]: 554 [\d.]+ <[^>]+>: Relay access denied.*from=<([^>]+)>') LOGIN_RE = re.compile(r'imap-login: Login: user=<([^>]+)>, method=(\S+), rip=([\d.:a-fA-F]+)') AUTH_FAIL_RE = re.compile(r'auth failed|authentication failure|invalid password', re.IGNORECASE) SENT_RE = re.compile(r'status=sent') @@ -100,8 +109,10 @@ def analyze_logs(lines): total_lines = len(lines) postscreen_rejects = Counter() + postscreen_reject_ips = Counter() dnsbl_hits = 0 relay_denied = [] + relay_denied_ips = Counter() logins = defaultdict(list) untrusted_logins = [] auth_failures = [] @@ -113,11 +124,13 @@ def analyze_logs(lines): for line in lines: if m := REJECT_RE.search(line): - postscreen_rejects[m.group(1)] += 1 + postscreen_reject_ips[m.group(1)] += 1 + postscreen_rejects[m.group(2)] += 1 if DNSBL_RE.search(line): dnsbl_hits += 1 if m := RELAY_DENIED_RE.search(line): - relay_denied.append(m.group(1)) + relay_denied_ips[m.group(1)] += 1 + relay_denied.append(m.group(2)) if m := LOGIN_RE.search(line): user, method, ip = m.group(1), m.group(2), m.group(3) logins[user].append(ip) @@ -137,6 +150,20 @@ def analyze_logs(lines): elif m := GENERIC_WARNING_RE.search(line): other_warnings[m.group(1).strip()] += 1 + # Union of IPs worth remembering across days: anything that got rejected, + # probed with junk protocol data, or tried to abuse the relay. Routine + # DNSBL-only connects are deliberately excluded -- too noisy/low-signal + # to track individually (see prior discussion on scanner noise). + suspicious_ips_today = Counter() + for ip, count in nonsmtp_by_ip.items(): + suspicious_ips_today[ip] += count + for ip, count in postscreen_reject_ips.items(): + suspicious_ips_today[ip] += count + for ip, count in relay_denied_ips.items(): + suspicious_ips_today[ip] += count + for user, method, ip in untrusted_logins: + suspicious_ips_today[ip] += 1 + facts = [] facts.append(f"Total log lines: {total_lines}") facts.append(f"Postscreen automatic rejections (bots/scanners blocked before reaching mailbox): {sum(postscreen_rejects.values())}") @@ -178,7 +205,65 @@ def analyze_logs(lines): needs_attention = bool(auth_failures) or bool(untrusted_logins) or bool(relay_denied) or bool(other_warnings) - return total_lines, "\n".join(facts), needs_attention + return total_lines, "\n".join(facts), needs_attention, suspicious_ips_today + + +def load_offender_state(): + if not os.path.exists(STATE_FILE): + debug_print(f"No existing state file at {STATE_FILE}, starting fresh.") + return {} + try: + with open(STATE_FILE, "r") as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + debug_print(f"Could not read state file ({e}), starting fresh.") + return {} + + +def save_offender_state(state): + try: + os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True) + with open(STATE_FILE, "w") as f: + json.dump(state, f, indent=2, sort_keys=True) + except OSError as e: + debug_print(f"Could not write state file: {e}") + + +def update_offender_state(state, suspicious_ips_today, report_date_str): + """ + Records which IPs were suspicious today, merges into the persisted + history, prunes IPs not seen recently, and returns (new_state, repeat_offenders). + repeat_offenders is a list of dicts sorted by distinct-day count descending. + """ + for ip, count in suspicious_ips_today.items(): + entry = state.setdefault(ip, {"days": [], "hit_counts": {}}) + if report_date_str not in entry["days"]: + entry["days"].append(report_date_str) + entry["hit_counts"][report_date_str] = entry["hit_counts"].get(report_date_str, 0) + count + entry["last_seen"] = report_date_str + + # Prune anything not seen within the retention window + cutoff = date.fromisoformat(report_date_str) - timedelta(days=OFFENDER_RETENTION_DAYS) + pruned_state = {} + for ip, entry in state.items(): + last_seen = date.fromisoformat(entry.get("last_seen", report_date_str)) + if last_seen >= cutoff: + pruned_state[ip] = entry + + repeat_offenders = [] + for ip, entry in pruned_state.items(): + distinct_days = len(entry["days"]) + if distinct_days >= REPEAT_OFFENDER_THRESHOLD_DAYS: + repeat_offenders.append({ + "ip": ip, + "distinct_days": distinct_days, + "first_seen": min(entry["days"]), + "last_seen": entry["last_seen"], + "total_hits": sum(entry["hit_counts"].values()), + }) + repeat_offenders.sort(key=lambda x: x["distinct_days"], reverse=True) + + return pruned_state, repeat_offenders def get_ai_summary(facts_text): @@ -267,7 +352,25 @@ def send_email_report(total_traffic, ai_summary, facts_text, needs_attention): def job(): print(f"\n--- Starting log analysis for {JOURNAL_UNIT} ---") logs = get_yesterdays_logs() - total_traffic, facts_text, needs_attention = analyze_logs(logs) + total_traffic, facts_text, needs_attention, suspicious_ips_today = analyze_logs(logs) + + report_date_str = (date.today() - timedelta(days=1)).isoformat() + state = load_offender_state() + state, repeat_offenders = update_offender_state(state, suspicious_ips_today, report_date_str) + save_offender_state(state) + + if repeat_offenders: + needs_attention = True + lines = [f"Repeat offenders (seen probing/rejected on {REPEAT_OFFENDER_THRESHOLD_DAYS}+ distinct days -- consider a firewall block):"] + for o in repeat_offenders: + lines.append( + f" - {o['ip']}: seen on {o['distinct_days']} distinct days " + f"(first {o['first_seen']}, last {o['last_seen']}), {o['total_hits']} total hits" + ) + facts_text = facts_text + "\n" + "\n".join(lines) + else: + facts_text = facts_text + f"\nRepeat offenders (seen on {REPEAT_OFFENDER_THRESHOLD_DAYS}+ distinct days): 0" + debug_print(f"Extracted facts:\n{facts_text}") summary = get_ai_summary(facts_text) send_email_report(total_traffic, summary, facts_text, needs_attention) diff --git a/systemd-daily-ai-summary.service.example b/systemd-daily-ai-summary.service.example index 1166f10..13f8a0a 100644 --- a/systemd-daily-ai-summary.service.example +++ b/systemd-daily-ai-summary.service.example @@ -16,6 +16,7 @@ ExecStart=/usr/bin/env docker run \ --log-driver=none \ --network=host \ --env-file=/virt/daily-ai-summary/env \ + --mount type=bind,src=/virt/daily-ai-summary/data,dst=/data \ --mount type=bind,src=/var/log/journal,dst=/var/log/journal,ro \ --mount type=bind,src=/run/log/journal,dst=/run/log/journal,ro \ --mount type=bind,src=/etc/machine-id,dst=/etc/machine-id,ro \