220 lines
10 KiB
Django/Jinja
220 lines
10 KiB
Django/Jinja
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
outfile="/var/lib/prometheus/node-exporter/pmg_smtp.prom"
|
|
tmpfile="$(mktemp)"
|
|
statefile="/var/lib/prometheus/node-exporter/pmg_smtp.state"
|
|
statetmp="$(mktemp)"
|
|
now_epoch="$(date +%s)"
|
|
window_seconds={{ smtp_log_window_seconds | int }}
|
|
|
|
if [ "${window_seconds}" -lt 60 ]; then
|
|
window_seconds=60
|
|
fi
|
|
|
|
to_int() {
|
|
local value="${1:-0}"
|
|
if [[ "${value}" =~ ^[0-9]+$ ]]; then
|
|
echo "${value}"
|
|
else
|
|
echo "0"
|
|
fi
|
|
}
|
|
|
|
get_established_count() {
|
|
local port="$1"
|
|
ss -Htn state established "( sport = :${port} )" 2>/dev/null | awk 'END { print NR + 0 }'
|
|
}
|
|
|
|
get_listen_up() {
|
|
local port="$1"
|
|
if ss -Hln "( sport = :${port} )" 2>/dev/null | awk 'END { exit(NR > 0 ? 0 : 1) }'; then
|
|
echo "1"
|
|
else
|
|
echo "0"
|
|
fi
|
|
}
|
|
|
|
service_up() {
|
|
local service="$1"
|
|
if systemctl is-active --quiet "${service}" 2>/dev/null; then
|
|
echo "1"
|
|
else
|
|
echo "0"
|
|
fi
|
|
}
|
|
|
|
last_cursor=""
|
|
total_smtp_connections=0
|
|
total_smtp_received=0
|
|
total_smtp_sent=0
|
|
total_smtp_deferred=0
|
|
total_smtp_bounced=0
|
|
total_smtp_rejected=0
|
|
total_smtp_tls_failures=0
|
|
prev_queue_total=0
|
|
prev_queue_ts=0
|
|
|
|
if [ -f "${statefile}" ]; then
|
|
while IFS='=' read -r key value; do
|
|
case "${key}" in
|
|
last_cursor) last_cursor="${value}" ;;
|
|
total_smtp_connections) total_smtp_connections="$(to_int "${value}")" ;;
|
|
total_smtp_received) total_smtp_received="$(to_int "${value}")" ;;
|
|
total_smtp_sent) total_smtp_sent="$(to_int "${value}")" ;;
|
|
total_smtp_deferred) total_smtp_deferred="$(to_int "${value}")" ;;
|
|
total_smtp_bounced) total_smtp_bounced="$(to_int "${value}")" ;;
|
|
total_smtp_rejected) total_smtp_rejected="$(to_int "${value}")" ;;
|
|
total_smtp_tls_failures) total_smtp_tls_failures="$(to_int "${value}")" ;;
|
|
prev_queue_total) prev_queue_total="$(to_int "${value}")" ;;
|
|
prev_queue_ts) prev_queue_ts="$(to_int "${value}")" ;;
|
|
esac
|
|
done < "${statefile}"
|
|
fi
|
|
|
|
queue_total="$(postqueue -j 2>/dev/null | awk 'END { print NR + 0 }')"
|
|
queue_deferred="$(postqueue -j 2>/dev/null | awk -F'"' '/"queue_name":/ { if ($4 == "deferred") c++ } END { print c + 0 }')"
|
|
if command -v jq >/dev/null 2>&1; then
|
|
oldest_queue_epoch="$(postqueue -j 2>/dev/null | jq -sr 'map(.arrival_time // empty) | if length == 0 then 0 else min end' 2>/dev/null || echo "0")"
|
|
else
|
|
oldest_queue_epoch="0"
|
|
fi
|
|
|
|
if [ "${oldest_queue_epoch}" -gt 0 ] 2>/dev/null; then
|
|
queue_oldest_age_seconds="$(( now_epoch - oldest_queue_epoch ))"
|
|
if [ "${queue_oldest_age_seconds}" -lt 0 ]; then
|
|
queue_oldest_age_seconds=0
|
|
fi
|
|
else
|
|
queue_oldest_age_seconds=0
|
|
fi
|
|
|
|
if [ "${prev_queue_ts}" -gt 0 ] && [ "${now_epoch}" -gt "${prev_queue_ts}" ]; then
|
|
queue_delta_seconds="$(( now_epoch - prev_queue_ts ))"
|
|
queue_drain_per_minute="$(awk -v prev="${prev_queue_total}" -v curr="${queue_total}" -v dt="${queue_delta_seconds}" 'BEGIN { d = prev - curr; if (dt <= 0 || d <= 0) print 0; else printf "%.3f", (d * 60.0) / dt }')"
|
|
else
|
|
queue_drain_per_minute="0"
|
|
fi
|
|
|
|
log_slice="$(journalctl --no-pager -o cat --since "-${window_seconds} sec" 2>/dev/null || true)"
|
|
smtp_connections_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/smtpd.*connect from' || true)"
|
|
smtp_received_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)"
|
|
smtp_sent_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=sent' || true)"
|
|
smtp_deferred_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=deferred' || true)"
|
|
smtp_bounced_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=bounced' || true)"
|
|
smtp_rejected_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)"
|
|
smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)"
|
|
|
|
journal_cursor_args=()
|
|
if [ -n "${last_cursor}" ]; then
|
|
journal_cursor_args+=(--after-cursor "${last_cursor}")
|
|
fi
|
|
journal_delta="$(journalctl --no-pager -o cat "${journal_cursor_args[@]}" 2>/dev/null || true)"
|
|
delta_smtp_connections="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/smtpd.*connect from' || true)"
|
|
delta_smtp_received="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)"
|
|
delta_smtp_sent="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=sent' || true)"
|
|
delta_smtp_deferred="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=deferred' || true)"
|
|
delta_smtp_bounced="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=bounced' || true)"
|
|
delta_smtp_rejected="$(printf '%s\n' "${journal_delta}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)"
|
|
delta_smtp_tls_failures="$(printf '%s\n' "${journal_delta}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)"
|
|
|
|
total_smtp_connections="$(( total_smtp_connections + delta_smtp_connections ))"
|
|
total_smtp_received="$(( total_smtp_received + delta_smtp_received ))"
|
|
total_smtp_sent="$(( total_smtp_sent + delta_smtp_sent ))"
|
|
total_smtp_deferred="$(( total_smtp_deferred + delta_smtp_deferred ))"
|
|
total_smtp_bounced="$(( total_smtp_bounced + delta_smtp_bounced ))"
|
|
total_smtp_rejected="$(( total_smtp_rejected + delta_smtp_rejected ))"
|
|
total_smtp_tls_failures="$(( total_smtp_tls_failures + delta_smtp_tls_failures ))"
|
|
|
|
current_cursor="$(journalctl --no-pager -n 1 -o cat --show-cursor 2>/dev/null | awk -F': ' '/^-- cursor: / { c = $2 } END { print c }')"
|
|
if [ -z "${current_cursor}" ]; then
|
|
current_cursor="${last_cursor}"
|
|
fi
|
|
|
|
{
|
|
echo "# HELP pmg_postfix_queue_messages_total Number of messages in Postfix queue."
|
|
echo "# TYPE pmg_postfix_queue_messages_total gauge"
|
|
echo "pmg_postfix_queue_messages_total ${queue_total}"
|
|
echo "# HELP pmg_postfix_queue_deferred_messages Number of deferred messages in Postfix queue."
|
|
echo "# TYPE pmg_postfix_queue_deferred_messages gauge"
|
|
echo "pmg_postfix_queue_deferred_messages ${queue_deferred}"
|
|
echo "# HELP pmg_postfix_queue_oldest_age_seconds Age of oldest queued message in seconds."
|
|
echo "# TYPE pmg_postfix_queue_oldest_age_seconds gauge"
|
|
echo "pmg_postfix_queue_oldest_age_seconds ${queue_oldest_age_seconds}"
|
|
echo "# HELP pmg_postfix_queue_drain_per_minute Queue drain rate in messages per minute (only positive drain)."
|
|
echo "# TYPE pmg_postfix_queue_drain_per_minute gauge"
|
|
echo "pmg_postfix_queue_drain_per_minute ${queue_drain_per_minute}"
|
|
echo "# HELP pmg_smtp_metrics_window_seconds Time window in seconds used for recent SMTP counters."
|
|
echo "# TYPE pmg_smtp_metrics_window_seconds gauge"
|
|
echo "pmg_smtp_metrics_window_seconds ${window_seconds}"
|
|
echo "# HELP pmg_smtp_connections_recent Number of SMTP connections observed in recent window."
|
|
echo "# TYPE pmg_smtp_connections_recent gauge"
|
|
echo "pmg_smtp_connections_recent ${smtp_connections_recent}"
|
|
echo "# HELP pmg_smtp_received_recent Number of received messages observed in recent window."
|
|
echo "# TYPE pmg_smtp_received_recent gauge"
|
|
echo "pmg_smtp_received_recent ${smtp_received_recent}"
|
|
echo "# HELP pmg_smtp_sent_recent Number of sent messages observed in recent window."
|
|
echo "# TYPE pmg_smtp_sent_recent gauge"
|
|
echo "pmg_smtp_sent_recent ${smtp_sent_recent}"
|
|
echo "# HELP pmg_smtp_deferred_recent Number of deferred deliveries observed in recent window."
|
|
echo "# TYPE pmg_smtp_deferred_recent gauge"
|
|
echo "pmg_smtp_deferred_recent ${smtp_deferred_recent}"
|
|
echo "# HELP pmg_smtp_bounced_recent Number of bounced deliveries observed in recent window."
|
|
echo "# TYPE pmg_smtp_bounced_recent gauge"
|
|
echo "pmg_smtp_bounced_recent ${smtp_bounced_recent}"
|
|
echo "# HELP pmg_smtp_rejected_recent Number of rejected SMTP attempts observed in recent window."
|
|
echo "# TYPE pmg_smtp_rejected_recent gauge"
|
|
echo "pmg_smtp_rejected_recent ${smtp_rejected_recent}"
|
|
echo "# HELP pmg_smtp_tls_failures_recent Number of TLS/STARTTLS failures observed in recent window."
|
|
echo "# TYPE pmg_smtp_tls_failures_recent gauge"
|
|
echo "pmg_smtp_tls_failures_recent ${smtp_tls_fail_recent}"
|
|
echo "# HELP pmg_smtp_connections_total Total SMTP connections seen in logs since metrics script state start."
|
|
echo "# TYPE pmg_smtp_connections_total counter"
|
|
echo "pmg_smtp_connections_total ${total_smtp_connections}"
|
|
echo "# HELP pmg_smtp_messages_total Total SMTP message events by direction/status since metrics script state start."
|
|
echo "# TYPE pmg_smtp_messages_total counter"
|
|
echo "pmg_smtp_messages_total{direction=\"received\",status=\"accepted\"} ${total_smtp_received}"
|
|
echo "pmg_smtp_messages_total{direction=\"sent\",status=\"sent\"} ${total_smtp_sent}"
|
|
echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"deferred\"} ${total_smtp_deferred}"
|
|
echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"bounced\"} ${total_smtp_bounced}"
|
|
echo "pmg_smtp_messages_total{direction=\"inbound\",status=\"rejected\"} ${total_smtp_rejected}"
|
|
echo "# HELP pmg_smtp_tls_failures_total Total TLS/STARTTLS failures seen in logs since metrics script state start."
|
|
echo "# TYPE pmg_smtp_tls_failures_total counter"
|
|
echo "pmg_smtp_tls_failures_total ${total_smtp_tls_failures}"
|
|
echo "# HELP pmg_smtp_port_listen_up SMTP listen status on port (1 up, 0 down)."
|
|
echo "# TYPE pmg_smtp_port_listen_up gauge"
|
|
echo "# HELP pmg_smtp_port_established_connections Established SMTP connections on port."
|
|
echo "# TYPE pmg_smtp_port_established_connections gauge"
|
|
echo "# HELP pmg_service_up PMG-related service status (1 up, 0 down)."
|
|
echo "# TYPE pmg_service_up gauge"
|
|
{% for port in smtp_ports %}
|
|
echo "pmg_smtp_port_listen_up{port=\"{{ port }}\"} $(get_listen_up {{ port }})"
|
|
echo "pmg_smtp_port_established_connections{port=\"{{ port }}\"} $(get_established_count {{ port }})"
|
|
{% endfor %}
|
|
echo "pmg_service_up{service=\"postfix\"} $(service_up postfix)"
|
|
echo "pmg_service_up{service=\"pmg-smtp-filter\"} $(service_up pmg-smtp-filter)"
|
|
echo "pmg_service_up{service=\"pmgpolicy\"} $(service_up pmgpolicy)"
|
|
echo "pmg_service_up{service=\"pmgdaemon\"} $(service_up pmgdaemon)"
|
|
echo "pmg_service_up{service=\"pmgmirror\"} $(service_up pmgmirror)"
|
|
echo "pmg_service_up{service=\"pmgtunnel\"} $(service_up pmgtunnel)"
|
|
} > "${tmpfile}"
|
|
|
|
mv "${tmpfile}" "${outfile}"
|
|
chmod 0644 "${outfile}"
|
|
|
|
{
|
|
echo "last_cursor=${current_cursor}"
|
|
echo "total_smtp_connections=${total_smtp_connections}"
|
|
echo "total_smtp_received=${total_smtp_received}"
|
|
echo "total_smtp_sent=${total_smtp_sent}"
|
|
echo "total_smtp_deferred=${total_smtp_deferred}"
|
|
echo "total_smtp_bounced=${total_smtp_bounced}"
|
|
echo "total_smtp_rejected=${total_smtp_rejected}"
|
|
echo "total_smtp_tls_failures=${total_smtp_tls_failures}"
|
|
echo "prev_queue_total=${queue_total}"
|
|
echo "prev_queue_ts=${now_epoch}"
|
|
} > "${statetmp}"
|
|
|
|
mv "${statetmp}" "${statefile}"
|
|
chmod 0644 "${statefile}"
|