Add additional PMG mail gateway hosts and enhance SMTP metrics script with state management and new metrics. Updated metrics output to include queue drain rate and total SMTP connections.
This commit is contained in:
Ruslan Piatrovich
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,8 @@ set -euo pipefail
|
||||
|
||||
outfile="/var/lib/prometheus/node-exporter/pmg_smtp.prom"
|
||||
tmpfile="$(mktemp)"
|
||||
statefile="/var/lib/prometheus/node-exporter/pmg_smtp.state"
|
||||
statetmp="$(mktemp)"
|
||||
now_epoch="$(date +%s)"
|
||||
window_seconds={{ smtp_log_window_seconds | int }}
|
||||
|
||||
@@ -10,6 +12,15 @@ if [ "${window_seconds}" -lt 60 ]; then
|
||||
window_seconds=60
|
||||
fi
|
||||
|
||||
to_int() {
|
||||
local value="${1:-0}"
|
||||
if [[ "${value}" =~ ^[0-9]+$ ]]; then
|
||||
echo "${value}"
|
||||
else
|
||||
echo "0"
|
||||
fi
|
||||
}
|
||||
|
||||
get_established_count() {
|
||||
local port="$1"
|
||||
ss -Htn state established "( sport = :${port} )" 2>/dev/null | awk 'END { print NR + 0 }'
|
||||
@@ -24,6 +35,43 @@ get_listen_up() {
|
||||
fi
|
||||
}
|
||||
|
||||
service_up() {
|
||||
local service="$1"
|
||||
if systemctl is-active --quiet "${service}" 2>/dev/null; then
|
||||
echo "1"
|
||||
else
|
||||
echo "0"
|
||||
fi
|
||||
}
|
||||
|
||||
last_cursor=""
|
||||
total_smtp_connections=0
|
||||
total_smtp_received=0
|
||||
total_smtp_sent=0
|
||||
total_smtp_deferred=0
|
||||
total_smtp_bounced=0
|
||||
total_smtp_rejected=0
|
||||
total_smtp_tls_failures=0
|
||||
prev_queue_total=0
|
||||
prev_queue_ts=0
|
||||
|
||||
if [ -f "${statefile}" ]; then
|
||||
while IFS='=' read -r key value; do
|
||||
case "${key}" in
|
||||
last_cursor) last_cursor="${value}" ;;
|
||||
total_smtp_connections) total_smtp_connections="$(to_int "${value}")" ;;
|
||||
total_smtp_received) total_smtp_received="$(to_int "${value}")" ;;
|
||||
total_smtp_sent) total_smtp_sent="$(to_int "${value}")" ;;
|
||||
total_smtp_deferred) total_smtp_deferred="$(to_int "${value}")" ;;
|
||||
total_smtp_bounced) total_smtp_bounced="$(to_int "${value}")" ;;
|
||||
total_smtp_rejected) total_smtp_rejected="$(to_int "${value}")" ;;
|
||||
total_smtp_tls_failures) total_smtp_tls_failures="$(to_int "${value}")" ;;
|
||||
prev_queue_total) prev_queue_total="$(to_int "${value}")" ;;
|
||||
prev_queue_ts) prev_queue_ts="$(to_int "${value}")" ;;
|
||||
esac
|
||||
done < "${statefile}"
|
||||
fi
|
||||
|
||||
queue_total="$(postqueue -j 2>/dev/null | awk 'END { print NR + 0 }')"
|
||||
queue_deferred="$(postqueue -j 2>/dev/null | awk -F'"' '/"queue_name":/ { if ($4 == "deferred") c++ } END { print c + 0 }')"
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
@@ -41,6 +89,13 @@ else
|
||||
queue_oldest_age_seconds=0
|
||||
fi
|
||||
|
||||
if [ "${prev_queue_ts}" -gt 0 ] && [ "${now_epoch}" -gt "${prev_queue_ts}" ]; then
|
||||
queue_delta_seconds="$(( now_epoch - prev_queue_ts ))"
|
||||
queue_drain_per_minute="$(awk -v prev="${prev_queue_total}" -v curr="${queue_total}" -v dt="${queue_delta_seconds}" 'BEGIN { d = prev - curr; if (dt <= 0 || d <= 0) print 0; else printf "%.3f", (d * 60.0) / dt }')"
|
||||
else
|
||||
queue_drain_per_minute="0"
|
||||
fi
|
||||
|
||||
log_slice="$(journalctl --no-pager -o cat --since "-${window_seconds} sec" 2>/dev/null || true)"
|
||||
smtp_connections_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/smtpd.*connect from' || true)"
|
||||
smtp_received_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)"
|
||||
@@ -50,6 +105,32 @@ smtp_bounced_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=bounced'
|
||||
smtp_rejected_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)"
|
||||
smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)"
|
||||
|
||||
journal_cursor_args=()
|
||||
if [ -n "${last_cursor}" ]; then
|
||||
journal_cursor_args+=(--after-cursor "${last_cursor}")
|
||||
fi
|
||||
journal_delta="$(journalctl --no-pager -o cat "${journal_cursor_args[@]}" 2>/dev/null || true)"
|
||||
delta_smtp_connections="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/smtpd.*connect from' || true)"
|
||||
delta_smtp_received="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)"
|
||||
delta_smtp_sent="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=sent' || true)"
|
||||
delta_smtp_deferred="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=deferred' || true)"
|
||||
delta_smtp_bounced="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=bounced' || true)"
|
||||
delta_smtp_rejected="$(printf '%s\n' "${journal_delta}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)"
|
||||
delta_smtp_tls_failures="$(printf '%s\n' "${journal_delta}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)"
|
||||
|
||||
total_smtp_connections="$(( total_smtp_connections + delta_smtp_connections ))"
|
||||
total_smtp_received="$(( total_smtp_received + delta_smtp_received ))"
|
||||
total_smtp_sent="$(( total_smtp_sent + delta_smtp_sent ))"
|
||||
total_smtp_deferred="$(( total_smtp_deferred + delta_smtp_deferred ))"
|
||||
total_smtp_bounced="$(( total_smtp_bounced + delta_smtp_bounced ))"
|
||||
total_smtp_rejected="$(( total_smtp_rejected + delta_smtp_rejected ))"
|
||||
total_smtp_tls_failures="$(( total_smtp_tls_failures + delta_smtp_tls_failures ))"
|
||||
|
||||
current_cursor="$(journalctl --no-pager -n 1 -o cat --show-cursor 2>/dev/null | awk -F': ' '/^-- cursor: / { c = $2 } END { print c }')"
|
||||
if [ -z "${current_cursor}" ]; then
|
||||
current_cursor="${last_cursor}"
|
||||
fi
|
||||
|
||||
{
|
||||
echo "# HELP pmg_postfix_queue_messages_total Number of messages in Postfix queue."
|
||||
echo "# TYPE pmg_postfix_queue_messages_total gauge"
|
||||
@@ -60,6 +141,9 @@ smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls)
|
||||
echo "# HELP pmg_postfix_queue_oldest_age_seconds Age of oldest queued message in seconds."
|
||||
echo "# TYPE pmg_postfix_queue_oldest_age_seconds gauge"
|
||||
echo "pmg_postfix_queue_oldest_age_seconds ${queue_oldest_age_seconds}"
|
||||
echo "# HELP pmg_postfix_queue_drain_per_minute Queue drain rate in messages per minute (only positive drain)."
|
||||
echo "# TYPE pmg_postfix_queue_drain_per_minute gauge"
|
||||
echo "pmg_postfix_queue_drain_per_minute ${queue_drain_per_minute}"
|
||||
echo "# HELP pmg_smtp_metrics_window_seconds Time window in seconds used for recent SMTP counters."
|
||||
echo "# TYPE pmg_smtp_metrics_window_seconds gauge"
|
||||
echo "pmg_smtp_metrics_window_seconds ${window_seconds}"
|
||||
@@ -84,15 +168,52 @@ smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls)
|
||||
echo "# HELP pmg_smtp_tls_failures_recent Number of TLS/STARTTLS failures observed in recent window."
|
||||
echo "# TYPE pmg_smtp_tls_failures_recent gauge"
|
||||
echo "pmg_smtp_tls_failures_recent ${smtp_tls_fail_recent}"
|
||||
echo "# HELP pmg_smtp_connections_total Total SMTP connections seen in logs since metrics script state start."
|
||||
echo "# TYPE pmg_smtp_connections_total counter"
|
||||
echo "pmg_smtp_connections_total ${total_smtp_connections}"
|
||||
echo "# HELP pmg_smtp_messages_total Total SMTP message events by direction/status since metrics script state start."
|
||||
echo "# TYPE pmg_smtp_messages_total counter"
|
||||
echo "pmg_smtp_messages_total{direction=\"received\",status=\"accepted\"} ${total_smtp_received}"
|
||||
echo "pmg_smtp_messages_total{direction=\"sent\",status=\"sent\"} ${total_smtp_sent}"
|
||||
echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"deferred\"} ${total_smtp_deferred}"
|
||||
echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"bounced\"} ${total_smtp_bounced}"
|
||||
echo "pmg_smtp_messages_total{direction=\"inbound\",status=\"rejected\"} ${total_smtp_rejected}"
|
||||
echo "# HELP pmg_smtp_tls_failures_total Total TLS/STARTTLS failures seen in logs since metrics script state start."
|
||||
echo "# TYPE pmg_smtp_tls_failures_total counter"
|
||||
echo "pmg_smtp_tls_failures_total ${total_smtp_tls_failures}"
|
||||
echo "# HELP pmg_smtp_port_listen_up SMTP listen status on port (1 up, 0 down)."
|
||||
echo "# TYPE pmg_smtp_port_listen_up gauge"
|
||||
echo "# HELP pmg_smtp_port_established_connections Established SMTP connections on port."
|
||||
echo "# TYPE pmg_smtp_port_established_connections gauge"
|
||||
echo "# HELP pmg_service_up PMG-related service status (1 up, 0 down)."
|
||||
echo "# TYPE pmg_service_up gauge"
|
||||
{% for port in smtp_ports %}
|
||||
echo "pmg_smtp_port_listen_up{port=\"{{ port }}\"} $(get_listen_up {{ port }})"
|
||||
echo "pmg_smtp_port_established_connections{port=\"{{ port }}\"} $(get_established_count {{ port }})"
|
||||
{% endfor %}
|
||||
echo "pmg_service_up{service=\"postfix\"} $(service_up postfix)"
|
||||
echo "pmg_service_up{service=\"pmg-smtp-filter\"} $(service_up pmg-smtp-filter)"
|
||||
echo "pmg_service_up{service=\"pmgpolicy\"} $(service_up pmgpolicy)"
|
||||
echo "pmg_service_up{service=\"pmgdaemon\"} $(service_up pmgdaemon)"
|
||||
echo "pmg_service_up{service=\"pmgmirror\"} $(service_up pmgmirror)"
|
||||
echo "pmg_service_up{service=\"pmgtunnel\"} $(service_up pmgtunnel)"
|
||||
} > "${tmpfile}"
|
||||
|
||||
mv "${tmpfile}" "${outfile}"
|
||||
chmod 0644 "${outfile}"
|
||||
|
||||
{
|
||||
echo "last_cursor=${current_cursor}"
|
||||
echo "total_smtp_connections=${total_smtp_connections}"
|
||||
echo "total_smtp_received=${total_smtp_received}"
|
||||
echo "total_smtp_sent=${total_smtp_sent}"
|
||||
echo "total_smtp_deferred=${total_smtp_deferred}"
|
||||
echo "total_smtp_bounced=${total_smtp_bounced}"
|
||||
echo "total_smtp_rejected=${total_smtp_rejected}"
|
||||
echo "total_smtp_tls_failures=${total_smtp_tls_failures}"
|
||||
echo "prev_queue_total=${queue_total}"
|
||||
echo "prev_queue_ts=${now_epoch}"
|
||||
} > "${statetmp}"
|
||||
|
||||
mv "${statetmp}" "${statefile}"
|
||||
chmod 0644 "${statefile}"
|
||||
|
||||
Reference in New Issue
Block a user