#!/usr/bin/env bash set -euo pipefail outfile="/var/lib/prometheus/node-exporter/pmg_smtp.prom" tmpfile="$(mktemp)" statefile="/var/lib/prometheus/node-exporter/pmg_smtp.state" statetmp="$(mktemp)" now_epoch="$(date +%s)" window_seconds={{ smtp_log_window_seconds | int }} if [ "${window_seconds}" -lt 60 ]; then window_seconds=60 fi to_int() { local value="${1:-0}" if [[ "${value}" =~ ^[0-9]+$ ]]; then echo "${value}" else echo "0" fi } get_established_count() { local port="$1" ss -Htn state established "( sport = :${port} )" 2>/dev/null | awk 'END { print NR + 0 }' } get_listen_up() { local port="$1" if ss -Hln "( sport = :${port} )" 2>/dev/null | awk 'END { exit(NR > 0 ? 0 : 1) }'; then echo "1" else echo "0" fi } service_up() { local service="$1" if systemctl is-active --quiet "${service}" 2>/dev/null; then echo "1" else echo "0" fi } last_cursor="" total_smtp_connections=0 total_smtp_received=0 total_smtp_sent=0 total_smtp_deferred=0 total_smtp_bounced=0 total_smtp_rejected=0 total_smtp_tls_failures=0 prev_queue_total=0 prev_queue_ts=0 if [ -f "${statefile}" ]; then while IFS='=' read -r key value; do case "${key}" in last_cursor) last_cursor="${value}" ;; total_smtp_connections) total_smtp_connections="$(to_int "${value}")" ;; total_smtp_received) total_smtp_received="$(to_int "${value}")" ;; total_smtp_sent) total_smtp_sent="$(to_int "${value}")" ;; total_smtp_deferred) total_smtp_deferred="$(to_int "${value}")" ;; total_smtp_bounced) total_smtp_bounced="$(to_int "${value}")" ;; total_smtp_rejected) total_smtp_rejected="$(to_int "${value}")" ;; total_smtp_tls_failures) total_smtp_tls_failures="$(to_int "${value}")" ;; prev_queue_total) prev_queue_total="$(to_int "${value}")" ;; prev_queue_ts) prev_queue_ts="$(to_int "${value}")" ;; esac done < "${statefile}" fi queue_total="$(postqueue -j 2>/dev/null | awk 'END { print NR + 0 }')" queue_deferred="$(postqueue -j 2>/dev/null | awk -F'"' '/"queue_name":/ { if ($4 == "deferred") c++ } END { print c + 0 }')" if command -v jq >/dev/null 2>&1; then oldest_queue_epoch="$(postqueue -j 2>/dev/null | jq -sr 'map(.arrival_time // empty) | if length == 0 then 0 else min end' 2>/dev/null || echo "0")" else oldest_queue_epoch="0" fi if [ "${oldest_queue_epoch}" -gt 0 ] 2>/dev/null; then queue_oldest_age_seconds="$(( now_epoch - oldest_queue_epoch ))" if [ "${queue_oldest_age_seconds}" -lt 0 ]; then queue_oldest_age_seconds=0 fi else queue_oldest_age_seconds=0 fi if [ "${prev_queue_ts}" -gt 0 ] && [ "${now_epoch}" -gt "${prev_queue_ts}" ]; then queue_delta_seconds="$(( now_epoch - prev_queue_ts ))" queue_drain_per_minute="$(awk -v prev="${prev_queue_total}" -v curr="${queue_total}" -v dt="${queue_delta_seconds}" 'BEGIN { d = prev - curr; if (dt <= 0 || d <= 0) print 0; else printf "%.3f", (d * 60.0) / dt }')" else queue_drain_per_minute="0" fi log_slice="$(journalctl --no-pager -o cat --since "-${window_seconds} sec" 2>/dev/null || true)" smtp_connections_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/smtpd.*connect from' || true)" smtp_received_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)" smtp_sent_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=sent' || true)" smtp_deferred_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=deferred' || true)" smtp_bounced_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=bounced' || true)" smtp_rejected_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)" smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)" journal_cursor_args=() if [ -n "${last_cursor}" ]; then journal_cursor_args+=(--after-cursor "${last_cursor}") fi journal_delta="$(journalctl --no-pager -o cat "${journal_cursor_args[@]}" 2>/dev/null || true)" delta_smtp_connections="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/smtpd.*connect from' || true)" delta_smtp_received="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)" delta_smtp_sent="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=sent' || true)" delta_smtp_deferred="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=deferred' || true)" delta_smtp_bounced="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=bounced' || true)" delta_smtp_rejected="$(printf '%s\n' "${journal_delta}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)" delta_smtp_tls_failures="$(printf '%s\n' "${journal_delta}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)" total_smtp_connections="$(( total_smtp_connections + delta_smtp_connections ))" total_smtp_received="$(( total_smtp_received + delta_smtp_received ))" total_smtp_sent="$(( total_smtp_sent + delta_smtp_sent ))" total_smtp_deferred="$(( total_smtp_deferred + delta_smtp_deferred ))" total_smtp_bounced="$(( total_smtp_bounced + delta_smtp_bounced ))" total_smtp_rejected="$(( total_smtp_rejected + delta_smtp_rejected ))" total_smtp_tls_failures="$(( total_smtp_tls_failures + delta_smtp_tls_failures ))" current_cursor="$(journalctl --no-pager -n 1 -o cat --show-cursor 2>/dev/null | awk -F': ' '/^-- cursor: / { c = $2 } END { print c }')" if [ -z "${current_cursor}" ]; then current_cursor="${last_cursor}" fi { echo "# HELP pmg_postfix_queue_messages_total Number of messages in Postfix queue." echo "# TYPE pmg_postfix_queue_messages_total gauge" echo "pmg_postfix_queue_messages_total ${queue_total}" echo "# HELP pmg_postfix_queue_deferred_messages Number of deferred messages in Postfix queue." echo "# TYPE pmg_postfix_queue_deferred_messages gauge" echo "pmg_postfix_queue_deferred_messages ${queue_deferred}" echo "# HELP pmg_postfix_queue_oldest_age_seconds Age of oldest queued message in seconds." echo "# TYPE pmg_postfix_queue_oldest_age_seconds gauge" echo "pmg_postfix_queue_oldest_age_seconds ${queue_oldest_age_seconds}" echo "# HELP pmg_postfix_queue_drain_per_minute Queue drain rate in messages per minute (only positive drain)." echo "# TYPE pmg_postfix_queue_drain_per_minute gauge" echo "pmg_postfix_queue_drain_per_minute ${queue_drain_per_minute}" echo "# HELP pmg_smtp_metrics_window_seconds Time window in seconds used for recent SMTP counters." echo "# TYPE pmg_smtp_metrics_window_seconds gauge" echo "pmg_smtp_metrics_window_seconds ${window_seconds}" echo "# HELP pmg_smtp_connections_recent Number of SMTP connections observed in recent window." echo "# TYPE pmg_smtp_connections_recent gauge" echo "pmg_smtp_connections_recent ${smtp_connections_recent}" echo "# HELP pmg_smtp_received_recent Number of received messages observed in recent window." echo "# TYPE pmg_smtp_received_recent gauge" echo "pmg_smtp_received_recent ${smtp_received_recent}" echo "# HELP pmg_smtp_sent_recent Number of sent messages observed in recent window." echo "# TYPE pmg_smtp_sent_recent gauge" echo "pmg_smtp_sent_recent ${smtp_sent_recent}" echo "# HELP pmg_smtp_deferred_recent Number of deferred deliveries observed in recent window." echo "# TYPE pmg_smtp_deferred_recent gauge" echo "pmg_smtp_deferred_recent ${smtp_deferred_recent}" echo "# HELP pmg_smtp_bounced_recent Number of bounced deliveries observed in recent window." echo "# TYPE pmg_smtp_bounced_recent gauge" echo "pmg_smtp_bounced_recent ${smtp_bounced_recent}" echo "# HELP pmg_smtp_rejected_recent Number of rejected SMTP attempts observed in recent window." echo "# TYPE pmg_smtp_rejected_recent gauge" echo "pmg_smtp_rejected_recent ${smtp_rejected_recent}" echo "# HELP pmg_smtp_tls_failures_recent Number of TLS/STARTTLS failures observed in recent window." echo "# TYPE pmg_smtp_tls_failures_recent gauge" echo "pmg_smtp_tls_failures_recent ${smtp_tls_fail_recent}" echo "# HELP pmg_smtp_connections_total Total SMTP connections seen in logs since metrics script state start." echo "# TYPE pmg_smtp_connections_total counter" echo "pmg_smtp_connections_total ${total_smtp_connections}" echo "# HELP pmg_smtp_messages_total Total SMTP message events by direction/status since metrics script state start." echo "# TYPE pmg_smtp_messages_total counter" echo "pmg_smtp_messages_total{direction=\"received\",status=\"accepted\"} ${total_smtp_received}" echo "pmg_smtp_messages_total{direction=\"sent\",status=\"sent\"} ${total_smtp_sent}" echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"deferred\"} ${total_smtp_deferred}" echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"bounced\"} ${total_smtp_bounced}" echo "pmg_smtp_messages_total{direction=\"inbound\",status=\"rejected\"} ${total_smtp_rejected}" echo "# HELP pmg_smtp_tls_failures_total Total TLS/STARTTLS failures seen in logs since metrics script state start." echo "# TYPE pmg_smtp_tls_failures_total counter" echo "pmg_smtp_tls_failures_total ${total_smtp_tls_failures}" echo "# HELP pmg_smtp_port_listen_up SMTP listen status on port (1 up, 0 down)." echo "# TYPE pmg_smtp_port_listen_up gauge" echo "# HELP pmg_smtp_port_established_connections Established SMTP connections on port." echo "# TYPE pmg_smtp_port_established_connections gauge" echo "# HELP pmg_service_up PMG-related service status (1 up, 0 down)." echo "# TYPE pmg_service_up gauge" {% for port in smtp_ports %} echo "pmg_smtp_port_listen_up{port=\"{{ port }}\"} $(get_listen_up {{ port }})" echo "pmg_smtp_port_established_connections{port=\"{{ port }}\"} $(get_established_count {{ port }})" {% endfor %} echo "pmg_service_up{service=\"postfix\"} $(service_up postfix)" echo "pmg_service_up{service=\"pmg-smtp-filter\"} $(service_up pmg-smtp-filter)" echo "pmg_service_up{service=\"pmgpolicy\"} $(service_up pmgpolicy)" echo "pmg_service_up{service=\"pmgdaemon\"} $(service_up pmgdaemon)" echo "pmg_service_up{service=\"pmgmirror\"} $(service_up pmgmirror)" echo "pmg_service_up{service=\"pmgtunnel\"} $(service_up pmgtunnel)" } > "${tmpfile}" mv "${tmpfile}" "${outfile}" chmod 0644 "${outfile}" { echo "last_cursor=${current_cursor}" echo "total_smtp_connections=${total_smtp_connections}" echo "total_smtp_received=${total_smtp_received}" echo "total_smtp_sent=${total_smtp_sent}" echo "total_smtp_deferred=${total_smtp_deferred}" echo "total_smtp_bounced=${total_smtp_bounced}" echo "total_smtp_rejected=${total_smtp_rejected}" echo "total_smtp_tls_failures=${total_smtp_tls_failures}" echo "prev_queue_total=${queue_total}" echo "prev_queue_ts=${now_epoch}" } > "${statetmp}" mv "${statetmp}" "${statefile}" chmod 0644 "${statefile}"