diff --git a/pillar/pmg.sls b/pillar/pmg.sls index ddadf4c..e8420ca 100644 --- a/pillar/pmg.sls +++ b/pillar/pmg.sls @@ -64,10 +64,18 @@ pmg: hostname: p-mailgw-x1-01 aliases: - p-mailgw-x1-01.lmru.tech + - ip: 10.80.38.153 + hostname: p-mailgw-x1-02 + aliases: + - p-mailgw-x1-02.lmru.tech - ip: 10.220.41.152 hostname: p-mailgw-x2-01 aliases: - p-mailgw-x2-01.lmru.tech + - ip: 10.220.41.153 + hostname: p-mailgw-x2-02 + aliases: + - p-mailgw-x2-02.lmru.tech wazuh: enabled: true diff --git a/pmg/files/pmg-smtp-dashboard.json b/pmg/files/pmg-smtp-dashboard.json new file mode 100644 index 0000000..79a7201 --- /dev/null +++ b/pmg/files/pmg-smtp-dashboard.json @@ -0,0 +1,1739 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "PMG SMTP monitoring: blackbox probes, postfix queue, throughput, recent-window counters and service health. Infra metrics live on a separate dashboard.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Average probe_success over 5 minutes for selected SMTP ports.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.9 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "avg(avg_over_time(probe_success{job=~\"pmg-smtp-(${smtp_port:pipe})\",hostname=~\"$hostname\"}[5m]))", + "legendFormat": "availability", + "range": true, + "refId": "A" + } + ], + "title": "SMTP Availability (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Blackbox exporter targets responding on :9115.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "green", + "value": 4 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum(up{job=\"pmg-blackbox\",hostname=~\"$hostname\"})", + "legendFormat": "blackbox up", + "range": true, + "refId": "A" + } + ], + "title": "Blackbox Exporters Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum(pmg_smtp_port_listen_up{job=\"pmg\",hostname=~\"$hostname\",port=~\"${smtp_port:pipe}\"})", + "legendFormat": "listen up", + "range": true, + "refId": "A" + } + ], + "title": "Ports Listening", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum(pmg_smtp_port_established_connections{job=\"pmg\",hostname=~\"$hostname\",port=~\"${smtp_port:pipe}\"})", + "legendFormat": "established", + "range": true, + "refId": "A" + } + ], + "title": "Established SMTP Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "max(pmg_postfix_queue_messages_total{job=\"pmg\",hostname=~\"$hostname\"})", + "legendFormat": "queue max", + "range": true, + "refId": "A" + } + ], + "title": "Queue Total (max)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3600 + }, + { + "color": "red", + "value": 14400 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "max(pmg_postfix_queue_oldest_age_seconds{job=\"pmg\",hostname=~\"$hostname\"})", + "legendFormat": "oldest", + "range": true, + "refId": "A" + } + ], + "title": "Oldest Queued Message", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 101, + "panels": [], + "title": "Blackbox Probes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "showPoints": "never" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 1, + "text": "UP" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "probe_success{job=~\"pmg-smtp-(${smtp_port:pipe})\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} :{{smtp_port}}", + "range": true, + "refId": "A" + } + ], + "title": "Probe Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "probe_duration_seconds{job=~\"pmg-smtp-(${smtp_port:pipe})\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} :{{smtp_port}}", + "range": true, + "refId": "A" + } + ], + "title": "Probe Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "probe_dns_lookup_time_seconds{job=~\"pmg-smtp-(${smtp_port:pipe})\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} :{{smtp_port}}", + "range": true, + "refId": "A" + } + ], + "title": "Probe DNS Lookup Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "showPoints": "never" + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "OK" + }, + "1": { + "color": "red", + "index": 1, + "text": "REGEX FAIL" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "probe_failed_due_to_regex{job=~\"pmg-smtp-(${smtp_port:pipe})\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} :{{smtp_port}}", + "range": true, + "refId": "A" + } + ], + "title": "Probe Failed Due to Regex", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 102, + "panels": [], + "title": "SMTP Ports", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "showPoints": "never" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 1, + "text": "LISTEN" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "pmg_smtp_port_listen_up{job=\"pmg\",hostname=~\"$hostname\",port=~\"${smtp_port:pipe}\"}", + "legendFormat": "{{hostname}} :{{port}}", + "range": true, + "refId": "A" + } + ], + "title": "Port Listen Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "pmg_smtp_port_established_connections{job=\"pmg\",hostname=~\"$hostname\",port=~\"${smtp_port:pipe}\"}", + "legendFormat": "{{hostname}} :{{port}}", + "range": true, + "refId": "A" + } + ], + "title": "Established Connections by Port", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 103, + "panels": [], + "title": "Postfix Queue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*oldest.*" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 29 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "pmg_postfix_queue_messages_total{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} total", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "pmg_postfix_queue_deferred_messages{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} deferred", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "pmg_postfix_queue_oldest_age_seconds{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} oldest age", + "range": true, + "refId": "C" + } + ], + "title": "Queue Size and Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 25, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 29 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "pmg_postfix_queue_drain_per_minute{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} drain/min", + "range": true, + "refId": "A" + } + ], + "title": "Queue Drain Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 104, + "panels": [], + "title": "Throughput (counter rates)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (hostname) (rate(pmg_smtp_messages_total{job=\"pmg\",hostname=~\"$hostname\",direction=\"received\",status=\"accepted\"}[5m])) * 60", + "legendFormat": "{{hostname}} accepted/min", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum by (hostname) (rate(pmg_smtp_messages_total{job=\"pmg\",hostname=~\"$hostname\",direction=\"sent\",status=\"sent\"}[5m])) * 60", + "legendFormat": "{{hostname}} sent/min", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum by (hostname) (rate(pmg_smtp_connections_total{job=\"pmg\",hostname=~\"$hostname\"}[5m])) * 60", + "legendFormat": "{{hostname}} connections/min", + "range": true, + "refId": "C" + } + ], + "title": "Accepted / Sent / Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (hostname) (rate(pmg_smtp_messages_total{job=\"pmg\",hostname=~\"$hostname\",direction=\"inbound\",status=\"rejected\"}[5m])) * 60", + "legendFormat": "{{hostname}} rejected/min", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum by (hostname) (rate(pmg_smtp_messages_total{job=\"pmg\",hostname=~\"$hostname\",direction=\"delivery\",status=\"deferred\"}[5m])) * 60", + "legendFormat": "{{hostname}} deferred/min", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum by (hostname) (rate(pmg_smtp_messages_total{job=\"pmg\",hostname=~\"$hostname\",direction=\"delivery\",status=\"bounced\"}[5m])) * 60", + "legendFormat": "{{hostname}} bounced/min", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum by (hostname) (rate(pmg_smtp_tls_failures_total{job=\"pmg\",hostname=~\"$hostname\"}[5m])) * 60", + "legendFormat": "{{hostname}} tls_failures/min", + "range": true, + "refId": "D" + } + ], + "title": "Rejected / Deferred / Bounced / TLS", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 105, + "panels": [], + "title": "Recent Window (log slice gauges)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Counters from journalctl over pmg_smtp_metrics_window_seconds (default 300s). Updated every minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "pmg_smtp_connections_recent{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} connections", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "pmg_smtp_received_recent{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} received", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "pmg_smtp_sent_recent{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} sent", + "range": true, + "refId": "C" + } + ], + "title": "Recent: Connections / Received / Sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "pmg_smtp_deferred_recent{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} deferred", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "pmg_smtp_bounced_recent{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} bounced", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "pmg_smtp_rejected_recent{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} rejected", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "pmg_smtp_tls_failures_recent{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} tls failures", + "range": true, + "refId": "D" + } + ], + "title": "Recent: Deferred / Bounced / Rejected / TLS", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 106, + "panels": [], + "title": "PMG Services", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 30, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "showPoints": "never" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 1, + "text": "UP" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "pmg_service_up{job=\"pmg\",hostname=~\"$hostname\"}", + "legendFormat": "{{hostname}} {{service}}", + "range": true, + "refId": "A" + } + ], + "title": "PMG Services Up", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 107, + "panels": [], + "title": "Scrape Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 1, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 64 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum(up{job=\"pmg\",hostname=~\"$hostname\"})", + "legendFormat": "node exporters", + "range": true, + "refId": "A" + } + ], + "title": "PMG Node Exporters Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 1, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 64 + }, + "id": 21, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum(up{job=\"pmg-blackbox\",hostname=~\"$hostname\"})", + "legendFormat": "blackbox", + "range": true, + "refId": "A" + } + ], + "title": "PMG Blackbox Exporters Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Log window size configured in Salt (pmg.smtp_monitoring.log_window_seconds).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "fixedColor": "text", + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 64 + }, + "id": 22, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "max(pmg_smtp_metrics_window_seconds{job=\"pmg\",hostname=~\"$hostname\"})", + "legendFormat": "window", + "range": true, + "refId": "A" + } + ], + "title": "Recent Metrics Window", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "pmg", + "smtp", + "itsec" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "itsec", + "value": "itsec" + }, + "hide": 0, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(probe_success{job=~\"pmg-smtp-(25|26)\"}, hostname)", + "hide": 0, + "includeAll": true, + "label": "Hostname", + "multi": true, + "name": "hostname", + "options": [], + "query": { + "query": "label_values(probe_success{job=~\"pmg-smtp-(25|26)\"}, hostname)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "25", + "26" + ], + "value": [ + "25", + "26" + ] + }, + "hide": 0, + "includeAll": true, + "label": "SMTP Port", + "multi": true, + "name": "smtp_port", + "options": [ + { + "selected": true, + "text": "25", + "value": "25" + }, + { + "selected": true, + "text": "26", + "value": "26" + } + ], + "query": "25,26", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "PMG SMTP Monitoring", + "uid": "pmg-smtp-itsec", + "version": 3, + "weekStart": "" +} diff --git a/pmg/files/pmg-smtp-metrics.sh.jinja b/pmg/files/pmg-smtp-metrics.sh.jinja index d4da827..9ba6e78 100644 --- a/pmg/files/pmg-smtp-metrics.sh.jinja +++ b/pmg/files/pmg-smtp-metrics.sh.jinja @@ -3,6 +3,8 @@ set -euo pipefail outfile="/var/lib/prometheus/node-exporter/pmg_smtp.prom" tmpfile="$(mktemp)" +statefile="/var/lib/prometheus/node-exporter/pmg_smtp.state" +statetmp="$(mktemp)" now_epoch="$(date +%s)" window_seconds={{ smtp_log_window_seconds | int }} @@ -10,6 +12,15 @@ if [ "${window_seconds}" -lt 60 ]; then window_seconds=60 fi +to_int() { + local value="${1:-0}" + if [[ "${value}" =~ ^[0-9]+$ ]]; then + echo "${value}" + else + echo "0" + fi +} + get_established_count() { local port="$1" ss -Htn state established "( sport = :${port} )" 2>/dev/null | awk 'END { print NR + 0 }' @@ -24,6 +35,43 @@ get_listen_up() { fi } +service_up() { + local service="$1" + if systemctl is-active --quiet "${service}" 2>/dev/null; then + echo "1" + else + echo "0" + fi +} + +last_cursor="" +total_smtp_connections=0 +total_smtp_received=0 +total_smtp_sent=0 +total_smtp_deferred=0 +total_smtp_bounced=0 +total_smtp_rejected=0 +total_smtp_tls_failures=0 +prev_queue_total=0 +prev_queue_ts=0 + +if [ -f "${statefile}" ]; then + while IFS='=' read -r key value; do + case "${key}" in + last_cursor) last_cursor="${value}" ;; + total_smtp_connections) total_smtp_connections="$(to_int "${value}")" ;; + total_smtp_received) total_smtp_received="$(to_int "${value}")" ;; + total_smtp_sent) total_smtp_sent="$(to_int "${value}")" ;; + total_smtp_deferred) total_smtp_deferred="$(to_int "${value}")" ;; + total_smtp_bounced) total_smtp_bounced="$(to_int "${value}")" ;; + total_smtp_rejected) total_smtp_rejected="$(to_int "${value}")" ;; + total_smtp_tls_failures) total_smtp_tls_failures="$(to_int "${value}")" ;; + prev_queue_total) prev_queue_total="$(to_int "${value}")" ;; + prev_queue_ts) prev_queue_ts="$(to_int "${value}")" ;; + esac + done < "${statefile}" +fi + queue_total="$(postqueue -j 2>/dev/null | awk 'END { print NR + 0 }')" queue_deferred="$(postqueue -j 2>/dev/null | awk -F'"' '/"queue_name":/ { if ($4 == "deferred") c++ } END { print c + 0 }')" if command -v jq >/dev/null 2>&1; then @@ -41,6 +89,13 @@ else queue_oldest_age_seconds=0 fi +if [ "${prev_queue_ts}" -gt 0 ] && [ "${now_epoch}" -gt "${prev_queue_ts}" ]; then + queue_delta_seconds="$(( now_epoch - prev_queue_ts ))" + queue_drain_per_minute="$(awk -v prev="${prev_queue_total}" -v curr="${queue_total}" -v dt="${queue_delta_seconds}" 'BEGIN { d = prev - curr; if (dt <= 0 || d <= 0) print 0; else printf "%.3f", (d * 60.0) / dt }')" +else + queue_drain_per_minute="0" +fi + log_slice="$(journalctl --no-pager -o cat --since "-${window_seconds} sec" 2>/dev/null || true)" smtp_connections_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/smtpd.*connect from' || true)" smtp_received_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)" @@ -50,6 +105,32 @@ smtp_bounced_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=bounced' smtp_rejected_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)" smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)" +journal_cursor_args=() +if [ -n "${last_cursor}" ]; then + journal_cursor_args+=(--after-cursor "${last_cursor}") +fi +journal_delta="$(journalctl --no-pager -o cat "${journal_cursor_args[@]}" 2>/dev/null || true)" +delta_smtp_connections="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/smtpd.*connect from' || true)" +delta_smtp_received="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)" +delta_smtp_sent="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=sent' || true)" +delta_smtp_deferred="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=deferred' || true)" +delta_smtp_bounced="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=bounced' || true)" +delta_smtp_rejected="$(printf '%s\n' "${journal_delta}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)" +delta_smtp_tls_failures="$(printf '%s\n' "${journal_delta}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)" + +total_smtp_connections="$(( total_smtp_connections + delta_smtp_connections ))" +total_smtp_received="$(( total_smtp_received + delta_smtp_received ))" +total_smtp_sent="$(( total_smtp_sent + delta_smtp_sent ))" +total_smtp_deferred="$(( total_smtp_deferred + delta_smtp_deferred ))" +total_smtp_bounced="$(( total_smtp_bounced + delta_smtp_bounced ))" +total_smtp_rejected="$(( total_smtp_rejected + delta_smtp_rejected ))" +total_smtp_tls_failures="$(( total_smtp_tls_failures + delta_smtp_tls_failures ))" + +current_cursor="$(journalctl --no-pager -n 1 -o cat --show-cursor 2>/dev/null | awk -F': ' '/^-- cursor: / { c = $2 } END { print c }')" +if [ -z "${current_cursor}" ]; then + current_cursor="${last_cursor}" +fi + { echo "# HELP pmg_postfix_queue_messages_total Number of messages in Postfix queue." echo "# TYPE pmg_postfix_queue_messages_total gauge" @@ -60,6 +141,9 @@ smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls) echo "# HELP pmg_postfix_queue_oldest_age_seconds Age of oldest queued message in seconds." echo "# TYPE pmg_postfix_queue_oldest_age_seconds gauge" echo "pmg_postfix_queue_oldest_age_seconds ${queue_oldest_age_seconds}" + echo "# HELP pmg_postfix_queue_drain_per_minute Queue drain rate in messages per minute (only positive drain)." + echo "# TYPE pmg_postfix_queue_drain_per_minute gauge" + echo "pmg_postfix_queue_drain_per_minute ${queue_drain_per_minute}" echo "# HELP pmg_smtp_metrics_window_seconds Time window in seconds used for recent SMTP counters." echo "# TYPE pmg_smtp_metrics_window_seconds gauge" echo "pmg_smtp_metrics_window_seconds ${window_seconds}" @@ -84,15 +168,52 @@ smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls) echo "# HELP pmg_smtp_tls_failures_recent Number of TLS/STARTTLS failures observed in recent window." echo "# TYPE pmg_smtp_tls_failures_recent gauge" echo "pmg_smtp_tls_failures_recent ${smtp_tls_fail_recent}" + echo "# HELP pmg_smtp_connections_total Total SMTP connections seen in logs since metrics script state start." + echo "# TYPE pmg_smtp_connections_total counter" + echo "pmg_smtp_connections_total ${total_smtp_connections}" + echo "# HELP pmg_smtp_messages_total Total SMTP message events by direction/status since metrics script state start." + echo "# TYPE pmg_smtp_messages_total counter" + echo "pmg_smtp_messages_total{direction=\"received\",status=\"accepted\"} ${total_smtp_received}" + echo "pmg_smtp_messages_total{direction=\"sent\",status=\"sent\"} ${total_smtp_sent}" + echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"deferred\"} ${total_smtp_deferred}" + echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"bounced\"} ${total_smtp_bounced}" + echo "pmg_smtp_messages_total{direction=\"inbound\",status=\"rejected\"} ${total_smtp_rejected}" + echo "# HELP pmg_smtp_tls_failures_total Total TLS/STARTTLS failures seen in logs since metrics script state start." + echo "# TYPE pmg_smtp_tls_failures_total counter" + echo "pmg_smtp_tls_failures_total ${total_smtp_tls_failures}" echo "# HELP pmg_smtp_port_listen_up SMTP listen status on port (1 up, 0 down)." echo "# TYPE pmg_smtp_port_listen_up gauge" echo "# HELP pmg_smtp_port_established_connections Established SMTP connections on port." echo "# TYPE pmg_smtp_port_established_connections gauge" + echo "# HELP pmg_service_up PMG-related service status (1 up, 0 down)." + echo "# TYPE pmg_service_up gauge" {% for port in smtp_ports %} echo "pmg_smtp_port_listen_up{port=\"{{ port }}\"} $(get_listen_up {{ port }})" echo "pmg_smtp_port_established_connections{port=\"{{ port }}\"} $(get_established_count {{ port }})" {% endfor %} + echo "pmg_service_up{service=\"postfix\"} $(service_up postfix)" + echo "pmg_service_up{service=\"pmg-smtp-filter\"} $(service_up pmg-smtp-filter)" + echo "pmg_service_up{service=\"pmgpolicy\"} $(service_up pmgpolicy)" + echo "pmg_service_up{service=\"pmgdaemon\"} $(service_up pmgdaemon)" + echo "pmg_service_up{service=\"pmgmirror\"} $(service_up pmgmirror)" + echo "pmg_service_up{service=\"pmgtunnel\"} $(service_up pmgtunnel)" } > "${tmpfile}" mv "${tmpfile}" "${outfile}" chmod 0644 "${outfile}" + +{ + echo "last_cursor=${current_cursor}" + echo "total_smtp_connections=${total_smtp_connections}" + echo "total_smtp_received=${total_smtp_received}" + echo "total_smtp_sent=${total_smtp_sent}" + echo "total_smtp_deferred=${total_smtp_deferred}" + echo "total_smtp_bounced=${total_smtp_bounced}" + echo "total_smtp_rejected=${total_smtp_rejected}" + echo "total_smtp_tls_failures=${total_smtp_tls_failures}" + echo "prev_queue_total=${queue_total}" + echo "prev_queue_ts=${now_epoch}" +} > "${statetmp}" + +mv "${statetmp}" "${statefile}" +chmod 0644 "${statefile}"