From c2b8235b0e3506d0d36ac626c5df84edd7c58740 Mon Sep 17 00:00:00 2001 From: Tomasz Kramkowski Date: Wed, 12 Apr 2017 11:44:39 +0200 Subject: dmarcstats --- dmarcstats.py | 193 +++++++++++++++++++++++++++++++++++++++++ dmarcstats.sh | 3 + templates/feedback/export.xml | 72 +++++++++++++++ templates/feedback/index.html | 49 +++++++++++ templates/feedback/report.html | 113 ++++++++++++++++++++++++ templates/index.html | 11 +++ templates/layout.html | 15 ++++ templates/record/index.html | 96 ++++++++++++++++++++ templates/stats/index.html | 10 +++ 9 files changed, 562 insertions(+) create mode 100755 dmarcstats.py create mode 100755 dmarcstats.sh create mode 100644 templates/feedback/export.xml create mode 100644 templates/feedback/index.html create mode 100644 templates/feedback/report.html create mode 100644 templates/index.html create mode 100644 templates/layout.html create mode 100644 templates/record/index.html create mode 100644 templates/stats/index.html diff --git a/dmarcstats.py b/dmarcstats.py new file mode 100755 index 0000000..67f9fdf --- /dev/null +++ b/dmarcstats.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +import sqlite3 +from os.path import join as pathjoin +from flask import Flask, g, render_template, request, Markup, url_for +from datetime import datetime +from multiprocessing.pool import ThreadPool +from socket import getfqdn +import pygal + +rg_style = pygal.style.Style(colors=('#00ff00', '#ff0000')) + +sv = Flask(__name__) +sv.config.from_object(__name__) + +sv.config.update({ + 'database': pathjoin(sv.root_path, 'dmarc.db'), + }) + +def connect_db(): + rv = sqlite3.connect(sv.config['database']) + c = rv.cursor() + assert(c.execute('PRAGMA user_version').fetchone()[0] == 1) + rv.row_factory = sqlite3.Row + return rv + +def get_db(): + if not hasattr(g, 'sqlite_db'): + g.sqlite_db = connect_db() + return g.sqlite_db + +@sv.teardown_appcontext +def close_db(error): + if hasattr(g, 'sqlite_db'): + g.sqlite_db.close() + +def format_isotime(value, sep=' ', timespec='auto'): + return datetime.utcfromtimestamp(value).isoformat(sep=sep, timespec=timespec) +sv.jinja_env.filters['isotime'] = format_isotime + +rlut = dict() + +def bulk_rdns_worker(addr): + return addr, getfqdn(addr) + +def bulk_rdns(addrs): + addrs = [a for a in addrs if a not in rlut.keys()] + if not addrs: + return + pool = ThreadPool(processes=min(len(addrs), 50)) + for addr, dname in pool.imap( + bulk_rdns_worker, + addrs, + chunksize=1): + rlut[addr] = dname + pool.close() + +def format_ipaddr(value): + try: + return rlut[value] + except KeyError: + return value +sv.jinja_env.filters['fqdn'] = format_ipaddr + +def format_tblclass(value): + return { + 'pass': 'success', + 'policy': 'info', + 'none': 'notaclass', + 'neutral': 'info', + 'softfail': 'warning', + 'temperror': 'warning', + 'fail': 'danger', + 'permerror': 'danger', + }[value] +sv.jinja_env.filters['tblclass'] = format_tblclass + +def format_alignment(value): + return { 'r': 'Relaxed', 's': 'Strict', None: 'Unknown' }[value] +sv.jinja_env.filters['alignment'] = format_alignment + +def order_header(name, dest, order, current): + icon = '△' + destorder = order + if order == current: + icon = '▲' + destorder = '{} desc'.format(order) + if '{} desc'.format(order) == current: + icon = '▼' + href = url_for(dest, order=destorder) + return '{} {}'.format(href, name, icon) + +def format_pctclass(value, maximum, td=None, tw=None): + td = td or maximum / 3 + tw = tw or maximum / 3 * 2 + if value < td: + return 'danger' + elif value < tw: + return 'warning' + else: + return 'success' +sv.jinja_env.filters['pctclass'] = format_pctclass + +@sv.route('/') +def index(): + return render_template('index.html') + +@sv.route('/feedback') +def feedback(): + db = get_db() + order = request.args.get('order', 'rm_date_begin').split() + if order[0] not in ('rm_org_name', 'rm_date_begin', 'rm_date_end', + 'nemails'): + order = 'rm_date_begin' + if len(order) == 2 and order[1] == 'desc': + order = ' '.join(order) + else: + order = order[0] + feedback = db.execute(''' +SELECT f.*, SUM(row_count) AS nemails, + SUM((row_pol_disposition = 'none') * row_count) AS pass_none, + SUM((row_pol_dkim = 'pass' OR row_pol_spf = 'pass') * row_count) AS pass_dmarc, + SUM((row_pol_dkim = 'pass') * row_count) AS pass_dkim, + SUM((row_pol_spf = 'pass') * row_count) AS pass_spf +FROM feedback AS f +JOIN record AS r USING (feedbackid) +GROUP BY feedbackid +ORDER BY {} +'''.format(order)).fetchall() + return render_template('feedback/index.html', feedback=feedback, + order=order, orderhdr=order_header) + +@sv.route('/feedback/') +def report(feedbackid=None): + db = get_db() + feedback = db.execute('SELECT * FROM feedback WHERE feedbackid = ?', + (feedbackid, )).fetchone() + records = db.execute('SELECT * FROM record WHERE feedbackid = ?', + (feedbackid, )).fetchall() + bulk_rdns([r['row_source_ip'] for r in records]) + total = dict( + records = sum(r['row_count'] for r in records), + none = sum(r['row_pol_disposition'] == 'none' + and r['row_count'] or 0 for r in records), + dmarc = sum((r['row_pol_dkim'] == 'pass' or + r['row_pol_spf'] == 'pass') + and r['row_count'] or 0 for r in records), + dkim = sum(r['row_pol_dkim'] == 'pass' + and r['row_count'] or 0 for r in records), + spf = sum(r['row_pol_spf'] == 'pass' + and r['row_count'] or 0 for r in records)) + return render_template('feedback/report.html', feedback=feedback, + records=records, total=total) + +@sv.route('/record/') +def record(recordid): + db = get_db() + record = db.execute('SELECT * FROM record WHERE recordid = ?', + (recordid, )).fetchone() + bulk_rdns([record['row_source_ip']]) + reasons = db.execute('SELECT * FROM row_pol_reason WHERE recordid = ?', + (recordid, )).fetchall() + resdkim = db.execute('SELECT * FROM res_dkim WHERE recordid = ?', + (recordid, )).fetchall() + resspf = db.execute('SELECT * FROM res_spf WHERE recordid = ?', + (recordid, )).fetchall() + return render_template('record/index.html', record=record, reasons=reasons, + resdkim=resdkim, resspf=resspf) + +@sv.route('/feedback/.xml') +def export(feedbackid): + db = get_db() + feedback = db.execute('SELECT * FROM feedback WHERE feedbackid = ?', + (feedbackid, )).fetchone() + records = db.execute('SELECT * FROM record WHERE feedbackid = ?', + (feedbackid, )).fetchall() + reasons = db.execute(''' +SELECT r.* FROM row_pol_reason AS r +JOIN record USING (recordid) +WHERE feedbackid = ?''', (feedbackid, )).fetchall() + resdkim = db.execute(''' +SELECT d.* FROM res_dkim AS d +JOIN record USING (recordid) +WHERE feedbackid = ?''', (feedbackid, )).fetchall() + resspf = db.execute(''' +SELECT s.* FROM res_spf AS s +JOIN record USING (recordid) +WHERE feedbackid = ?''', (feedbackid, )).fetchall() + return render_template('feedback/export.xml', feedback=feedback, + records=records, reasons=reasons, resdkim=resdkim, resspf=resspf) + +@sv.route('/stats') +def stats(): + return render_template('stats/index.html') diff --git a/dmarcstats.sh b/dmarcstats.sh new file mode 100755 index 0000000..5194c65 --- /dev/null +++ b/dmarcstats.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +FLASK_DEBUG=$1 FLASK_APP=dmarcstats.py python -m flask run diff --git a/templates/feedback/export.xml b/templates/feedback/export.xml new file mode 100644 index 0000000..ad556d2 --- /dev/null +++ b/templates/feedback/export.xml @@ -0,0 +1,72 @@ + + + + {{ feedback.rm_org_name }} + {{ feedback.rm_email }} + {% if feedback.rm_extra_contact_info %} + {{ feedback.rm_extra_contact_info }} + {% endif %} + {{ feedback.rm_report_id }} + + {{ feedback.rm_date_begin }} + {{ feedback.rm_date_end }} + + + + {{ feedback.pp_domain }} + {{ feedback.pp_adkim }} + {{ feedback.pp_aspf }} +

{{ feedback.pp_p }}

+ {{ feedback.pp_sp }} + {{ feedback.pp_pct }} +
+ {% for r in records %} + + + {{ r.row_source_ip }} + {{ r.row_count }} + {% if r.row_pol_disposition and r.row_pol_dkim and r.row_pol_spf %} + + {{ r.row_pol_disposition }} + {{ r.row_pol_dkim }} + {{ r.row_pol_spf }} + {% for rs in reasons if rs.recordid == r.recordid %} + + {{ rs.type }} + {% if rs.comment %} + {{ rs.comment }} + {% endif %} + + {% endfor %} + + {% endif %} + + + {% if r.ids_envelope_to %} + {{ r.ids_envelope_to }} + {% endif %} + {{ r.ids_header_from }} + + + {% for d in resdkim if d.recordid == r.recordid %} + + {{ d.domain }} + {% if d.selector %} + {{ d.selector }} + {% endif %} + {{ d.result }} + {% if d.human_result %} + {{ d.human_result }} + {% endif %} + + {% endfor %} + {% for s in resspf if s.recordid == r.recordid %} + + {{ s.domain }} + {{ s.result }} + + {% endfor %} + + + {% endfor %} +
diff --git a/templates/feedback/index.html b/templates/feedback/index.html new file mode 100644 index 0000000..617c758 --- /dev/null +++ b/templates/feedback/index.html @@ -0,0 +1,49 @@ +{% extends "layout.html" %} +{% block title %}Feedback{% endblock %} +{% block content %} + +

Feedback

+ + + + + + + + + + + + + {% for item in feedback %} + + + + + + + + + + + + + + {% endfor %} +
{{ orderhdr('Organisation Name', 'feedback', 'rm_org_name', order)|safe }}{{ orderhdr('Date Begin', 'feedback', 'rm_date_begin', order)|safe }}{{ orderhdr('Date End', 'feedback', 'rm_date_end', order)|safe }}{{ orderhdr('Messages', 'feedback', 'nemails', order)|safe }}NonePassDKIMSPF
{{ item.rm_org_name }}{{ item.nemails }} + {{ (item.pass_none / item.nemails * 100)|round|int }}% + + {{ (item.pass_dmarc / item.nemails * 100)|round|int }}% + + {{ (item.pass_dkim / item.nemails * 100)|round|int }}% + + {{ (item.pass_spf / item.nemails * 100)|round|int }}% + + + Report + +
+{% endblock %} diff --git a/templates/feedback/report.html b/templates/feedback/report.html new file mode 100644 index 0000000..141c823 --- /dev/null +++ b/templates/feedback/report.html @@ -0,0 +1,113 @@ +{% extends "layout.html" %} +{% block title %}Report{% endblock %} +{% block content %} + +

Report

+
+
+

Report Metadata

+
+
Organisation Name
{{ feedback.rm_org_name }}
+
Email
{{ feedback.rm_email }}
+
Extra Contact Info
{{ feedback.rm_extra_contact_info }}
+
Report ID
{{ feedback.rm_report_id }}
+
Date Range
+ + to + +
+
+
+
+

Policy Published

+
+
Domain
{{ feedback.pp_domain }}
+
DKIM Alignment
{{ feedback.pp_adkim|alignment }}
+
SPF Alignment
{{ feedback.pp_aspf|alignment }}
+
Policy
{{ feedback.pp_p|capitalize }}
+
Subdomain Policy
{{ feedback.pp_sp|capitalize }}
+
+
+
+

Statistics

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Total%
Messages{{ total.records }}
Disposition None{{ total.none }}{{ (total.none / total.records * 100)|round|int }}%
DKIM or SPF Pass{{ total.dmarc }}{{ (total.dmarc / total.records * 100)|round|int }}%
DKIM Pass{{ total.dkim }}{{ (total.dkim / total.records * 100)|round|int }}%
SPF Pass{{ total.spf }}{{ (total.spf / total.records * 100)|round|int }}%
+
+
+

Utilities

+ +
+
+
+
+

Records

+ + + + + + + + + + + + {% for item in records %} + + + + + + + + + + + {% endfor %} +
SourceCountDispositionDKIMSPFEnvelope ToHeader From
{{ item['row_source_ip']|fqdn }}{{ item['row_count'] }}{{ item['row_pol_disposition'] }}{{ item['row_pol_dkim'] }}{{ item['row_pol_spf'] }}{{ item['ids_envelope_to'] }}{{ item['ids_header_from'] }} + + Record + +
+ {% endblock %} +
+
diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..1e0364f --- /dev/null +++ b/templates/index.html @@ -0,0 +1,11 @@ +{% extends "layout.html" %} +{% block title %}Index{% endblock %} +{% block content %} + + +{% endblock %} diff --git a/templates/layout.html b/templates/layout.html new file mode 100644 index 0000000..d7ca750 --- /dev/null +++ b/templates/layout.html @@ -0,0 +1,15 @@ + + + + {% block head %} + + + {% block title %}{% endblock %} - DMARC Statistics + + {% endblock %} + + + +
{% block content %}{% endblock %}
+ + diff --git a/templates/record/index.html b/templates/record/index.html new file mode 100644 index 0000000..ace1995 --- /dev/null +++ b/templates/record/index.html @@ -0,0 +1,96 @@ +{% extends "layout.html" %} +{% block title %}Record{% endblock %} +{% block content %} + +

Record

+
+
+

Record Details

+
+
Source IP
{{ record.row_source_ip|fqdn }}
+
Count
{{ record.row_count }}
+
+

Policy Evaluated

+
+
Disposition
{{ record.row_pol_disposition }}
+
DKIM
+ + {{ record.row_pol_dkim }} + +
+
SPF
+ + {{ record.row_pol_spf }} + +
+
+ {% if reasons %} +

Reasons

+ + + + + + {% for item in reasons %} + + + + + {% endfor %} +
TypeComment
{{ item['type'] }}{{ item['comment'] }}
+ {% endif %} +
+
+

Identifiers

+
+
Envelope To
{{ record.ids_envelope_to }}
+
Header From
{{ record.ids_header_from }}
+
+
+
+ +
+ {% if resdkim %} +
+

DKIM Results

+ + + + + + + + {% for item in resdkim %} + + + + + + + {% endfor %} +
DomainSelectorResultHuman Result
{{ item['domain'] }}{{ item['selector'] }}{{ item['result'] }}{{ item['human_result'] }}
+
+ {% endif %} + +
+

SPF Results

+ + + + + + {% for item in resspf %} + + + + + {% endfor %} +
DomainResult
{{ item['domain'] }}{{ item['result'] }}
+ {% endblock %} +
+
diff --git a/templates/stats/index.html b/templates/stats/index.html new file mode 100644 index 0000000..aab0f4e --- /dev/null +++ b/templates/stats/index.html @@ -0,0 +1,10 @@ +{% extends "layout.html" %} +{% block title %}Statistics{% endblock %} +{% block content %} + +

Statistics

+

Nothing here yet

+{% endblock %} -- cgit v1.2.3-54-g00ecf