Add configuration for Prometheus node exporter and SMTP monitoring in PMG. Updated README and added verification checks for new metrics.

README.md

@@ -8,6 +8,8 @@
 - `pillar/pmg.sls` — переменные и параметры PMG.
 - `pmg/*.sls` — состояния:
   - `common` (базовые пакеты),
+  - `node_exporter` (установка и включение Prometheus node exporter),
+  - `smtp_monitoring` (SMTP-метрики: blackbox probe для 25/26 и postfix queue/connection gauges),
   - `ssh_hardening`,
   - `custom_check` (кастомная AV-проверка PMG через KESL),
   - `security_agents` (Wazuh/Qualys),
@@ -50,6 +52,8 @@ make check MINION_ID=p-mailgw-x1-01.lmru.tech
 - `pmg.ssh_password_authentication`
 - `pmg.journald_system_max_use`
 - `pmg.journald_max_file_sec`
+- `pmg.node_exporter.*`
+- `pmg.smtp_monitoring.*`
 - `pmg.pmg_services`
 - `pmg.update_dist_upgrade`
 - `pmg.update_auto_reboot`
@@ -60,6 +64,15 @@ make check MINION_ID=p-mailgw-x1-01.lmru.tech
 - `pmg.wazuh.*`
 - `pmg.qualys.*`
 
+`pmg.node_exporter.args` по умолчанию включает textfile collector:
+`--collector.textfile.directory=/var/lib/prometheus/node-exporter`.
+
+`pmg.smtp_monitoring` публикует расширенные SMTP метрики в node_exporter:
+- queue total/deferred + age oldest message;
+- SMTP listen/established для портов 25/26;
+- recent window counters (`pmg.smtp_monitoring.log_window_seconds`) по логам postfix:
+  sent, received, deferred, bounced, rejected, tls failures, smtp connections.
+
 ## Отдельное обновление PMG
 
 Перед обновлением:

pillar/pmg.sls

@@ -12,6 +12,23 @@ pmg:
   journald_system_max_use: 1G
   journald_max_file_sec: 1month
 
+  node_exporter:
+    enabled: true
+    package: prometheus-node-exporter
+    service: prometheus-node-exporter
+    textfile_directory: /var/lib/prometheus/node-exporter
+    args: --collector.textfile.directory=/var/lib/prometheus/node-exporter
+
+  smtp_monitoring:
+    enabled: true
+    log_window_seconds: 300
+    ports:
+      - 25
+      - 26
+    blackbox:
+      package: prometheus-blackbox-exporter
+      service: prometheus-blackbox-exporter
+
   pmg_services:
     - postfix
     - pmg-smtp-filter

pmg/custom_check.sls

@@ -30,7 +30,7 @@ pmg-custom-check-ksc-superuser-sudoers:
     - group: root
     - mode: '0440'
     - contents:
-      - {{ ksc_superuser_name }} ALL = (ALL) NOPASSWD: ALL
+      - '{{ ksc_superuser_name }} ALL = (ALL) NOPASSWD: ALL'
     - check_cmd: visudo -cf
     - require:
       - pkg: pmg-custom-check-ksc-superuser-sudo-package

pmg/files/blackbox.yml.jinja

@@ -0,0 +1,12 @@
+modules:
+{% for port in smtp_ports %}
+  smtp_{{ port }}:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      preferred_ip_protocol: ip4
+      query_response:
+        - expect: "^220[ -]"
+        - send: "EHLO blackbox.local"
+        - expect: "^250[ -]"
+{% endfor %}

pmg/files/pmg-smtp-metrics.service.jinja

@@ -0,0 +1,8 @@
+[Unit]
+Description=Generate PMG SMTP metrics for node_exporter textfile collector
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/pmg-smtp-metrics.sh

pmg/files/pmg-smtp-metrics.sh.jinja

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+outfile="/var/lib/prometheus/node-exporter/pmg_smtp.prom"
+tmpfile="$(mktemp)"
+now_epoch="$(date +%s)"
+window_seconds={{ smtp_log_window_seconds | int }}
+
+if [ "${window_seconds}" -lt 60 ]; then
+  window_seconds=60
+fi
+
+get_established_count() {
+  local port="$1"
+  ss -Htn state established "( sport = :${port} )" 2>/dev/null | awk 'END { print NR + 0 }'
+}
+
+get_listen_up() {
+  local port="$1"
+  if ss -Hln "( sport = :${port} )" 2>/dev/null | awk 'END { exit(NR > 0 ? 0 : 1) }'; then
+    echo "1"
+  else
+    echo "0"
+  fi
+}
+
+queue_total="$(postqueue -j 2>/dev/null | awk 'END { print NR + 0 }')"
+queue_deferred="$(postqueue -j 2>/dev/null | awk -F'"' '/"queue_name":/ { if ($4 == "deferred") c++ } END { print c + 0 }')"
+if command -v jq >/dev/null 2>&1; then
+  oldest_queue_epoch="$(postqueue -j 2>/dev/null | jq -sr 'map(.arrival_time // empty) | if length == 0 then 0 else min end' 2>/dev/null || echo "0")"
+else
+  oldest_queue_epoch="0"
+fi
+
+if [ "${oldest_queue_epoch}" -gt 0 ] 2>/dev/null; then
+  queue_oldest_age_seconds="$(( now_epoch - oldest_queue_epoch ))"
+  if [ "${queue_oldest_age_seconds}" -lt 0 ]; then
+    queue_oldest_age_seconds=0
+  fi
+else
+  queue_oldest_age_seconds=0
+fi
+
+log_slice="$(journalctl --no-pager -o cat --since "-${window_seconds} sec" 2>/dev/null || true)"
+smtp_connections_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/smtpd.*connect from' || true)"
+smtp_received_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)"
+smtp_sent_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=sent' || true)"
+smtp_deferred_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=deferred' || true)"
+smtp_bounced_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=bounced' || true)"
+smtp_rejected_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)"
+smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)"
+
+{
+  echo "# HELP pmg_postfix_queue_messages_total Number of messages in Postfix queue."
+  echo "# TYPE pmg_postfix_queue_messages_total gauge"
+  echo "pmg_postfix_queue_messages_total ${queue_total}"
+  echo "# HELP pmg_postfix_queue_deferred_messages Number of deferred messages in Postfix queue."
+  echo "# TYPE pmg_postfix_queue_deferred_messages gauge"
+  echo "pmg_postfix_queue_deferred_messages ${queue_deferred}"
+  echo "# HELP pmg_postfix_queue_oldest_age_seconds Age of oldest queued message in seconds."
+  echo "# TYPE pmg_postfix_queue_oldest_age_seconds gauge"
+  echo "pmg_postfix_queue_oldest_age_seconds ${queue_oldest_age_seconds}"
+  echo "# HELP pmg_smtp_metrics_window_seconds Time window in seconds used for recent SMTP counters."
+  echo "# TYPE pmg_smtp_metrics_window_seconds gauge"
+  echo "pmg_smtp_metrics_window_seconds ${window_seconds}"
+  echo "# HELP pmg_smtp_connections_recent Number of SMTP connections observed in recent window."
+  echo "# TYPE pmg_smtp_connections_recent gauge"
+  echo "pmg_smtp_connections_recent ${smtp_connections_recent}"
+  echo "# HELP pmg_smtp_received_recent Number of received messages observed in recent window."
+  echo "# TYPE pmg_smtp_received_recent gauge"
+  echo "pmg_smtp_received_recent ${smtp_received_recent}"
+  echo "# HELP pmg_smtp_sent_recent Number of sent messages observed in recent window."
+  echo "# TYPE pmg_smtp_sent_recent gauge"
+  echo "pmg_smtp_sent_recent ${smtp_sent_recent}"
+  echo "# HELP pmg_smtp_deferred_recent Number of deferred deliveries observed in recent window."
+  echo "# TYPE pmg_smtp_deferred_recent gauge"
+  echo "pmg_smtp_deferred_recent ${smtp_deferred_recent}"
+  echo "# HELP pmg_smtp_bounced_recent Number of bounced deliveries observed in recent window."
+  echo "# TYPE pmg_smtp_bounced_recent gauge"
+  echo "pmg_smtp_bounced_recent ${smtp_bounced_recent}"
+  echo "# HELP pmg_smtp_rejected_recent Number of rejected SMTP attempts observed in recent window."
+  echo "# TYPE pmg_smtp_rejected_recent gauge"
+  echo "pmg_smtp_rejected_recent ${smtp_rejected_recent}"
+  echo "# HELP pmg_smtp_tls_failures_recent Number of TLS/STARTTLS failures observed in recent window."
+  echo "# TYPE pmg_smtp_tls_failures_recent gauge"
+  echo "pmg_smtp_tls_failures_recent ${smtp_tls_fail_recent}"
+  echo "# HELP pmg_smtp_port_listen_up SMTP listen status on port (1 up, 0 down)."
+  echo "# TYPE pmg_smtp_port_listen_up gauge"
+  echo "# HELP pmg_smtp_port_established_connections Established SMTP connections on port."
+  echo "# TYPE pmg_smtp_port_established_connections gauge"
+{% for port in smtp_ports %}
+  echo "pmg_smtp_port_listen_up{port=\"{{ port }}\"} $(get_listen_up {{ port }})"
+  echo "pmg_smtp_port_established_connections{port=\"{{ port }}\"} $(get_established_count {{ port }})"
+{% endfor %}
+} > "${tmpfile}"
+
+mv "${tmpfile}" "${outfile}"
+chmod 0644 "${outfile}"

pmg/files/pmg-smtp-metrics.timer.jinja

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run PMG SMTP metrics exporter script every minute
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec=1min
+AccuracySec=15s
+Unit=pmg-smtp-metrics.service
+
+[Install]
+WantedBy=timers.target

pmg/init.sls

@@ -1,5 +1,7 @@
 include:
   - pmg.common
+  - pmg.node_exporter
+  - pmg.smtp_monitoring
   - pmg.ssh_hardening
   - pmg.custom_check
   - pmg.security_agents

pmg/node_exporter.sls

@@ -0,0 +1,39 @@
+{% set node_exporter_enabled = salt['pillar.get']('pmg:node_exporter:enabled', True) %}
+{% set node_exporter_package = salt['pillar.get']('pmg:node_exporter:package', 'prometheus-node-exporter') %}
+{% set node_exporter_service = salt['pillar.get']('pmg:node_exporter:service', 'prometheus-node-exporter') %}
+{% set node_exporter_textfile_dir = salt['pillar.get']('pmg:node_exporter:textfile_directory', '/var/lib/prometheus/node-exporter') %}
+{% set node_exporter_args = salt['pillar.get']('pmg:node_exporter:args', '--collector.textfile.directory=' ~ node_exporter_textfile_dir) %}
+
+{% if node_exporter_enabled %}
+pmg-node-exporter-package:
+  pkg.installed:
+    - name: {{ node_exporter_package }}
+
+pmg-node-exporter-textfile-dir:
+  file.directory:
+    - name: {{ node_exporter_textfile_dir }}
+    - user: root
+    - group: root
+    - mode: "0755"
+
+pmg-node-exporter-defaults:
+  file.managed:
+    - name: /etc/default/prometheus-node-exporter
+    - user: root
+    - group: root
+    - mode: "0644"
+    - contents:
+      - "# Managed by Salt."
+      - "ARGS=\"{{ node_exporter_args }}\""
+
+pmg-node-exporter-service:
+  service.running:
+    - name: {{ node_exporter_service }}
+    - enable: true
+    - require:
+      - pkg: pmg-node-exporter-package
+      - file: pmg-node-exporter-textfile-dir
+      - file: pmg-node-exporter-defaults
+    - watch:
+      - file: pmg-node-exporter-defaults
+{% endif %}

pmg/smtp_monitoring.sls

@@ -0,0 +1,94 @@
+{% set smtp_monitoring_enabled = salt['pillar.get']('pmg:smtp_monitoring:enabled', True) %}
+{% set blackbox_package = salt['pillar.get']('pmg:smtp_monitoring:blackbox:package', 'prometheus-blackbox-exporter') %}
+{% set blackbox_service = salt['pillar.get']('pmg:smtp_monitoring:blackbox:service', 'prometheus-blackbox-exporter') %}
+{% set smtp_ports = salt['pillar.get']('pmg:smtp_monitoring:ports', [25, 26]) %}
+{% set smtp_log_window_seconds = salt['pillar.get']('pmg:smtp_monitoring:log_window_seconds', 300) %}
+
+{% if smtp_monitoring_enabled %}
+pmg-smtp-monitoring-packages:
+  pkg.installed:
+    - pkgs:
+      - {{ blackbox_package }}
+      - iproute2
+      - jq
+
+pmg-smtp-monitoring-metrics-dir:
+  file.directory:
+    - name: /var/lib/prometheus/node-exporter
+    - user: root
+    - group: root
+    - mode: "0755"
+
+pmg-smtp-monitoring-script:
+  file.managed:
+    - name: /usr/local/bin/pmg-smtp-metrics.sh
+    - source: salt://pmg/files/pmg-smtp-metrics.sh.jinja
+    - template: jinja
+    - context:
+        smtp_ports: {{ smtp_ports }}
+        smtp_log_window_seconds: {{ smtp_log_window_seconds }}
+    - user: root
+    - group: root
+    - mode: "0755"
+    - require:
+      - file: pmg-smtp-monitoring-metrics-dir
+
+pmg-smtp-monitoring-service:
+  file.managed:
+    - name: /etc/systemd/system/pmg-smtp-metrics.service
+    - source: salt://pmg/files/pmg-smtp-metrics.service.jinja
+    - user: root
+    - group: root
+    - mode: "0644"
+
+pmg-smtp-monitoring-timer:
+  file.managed:
+    - name: /etc/systemd/system/pmg-smtp-metrics.timer
+    - source: salt://pmg/files/pmg-smtp-metrics.timer.jinja
+    - user: root
+    - group: root
+    - mode: "0644"
+
+pmg-smtp-monitoring-blackbox-config:
+  file.managed:
+    - name: /etc/prometheus/blackbox.yml
+    - source: salt://pmg/files/blackbox.yml.jinja
+    - template: jinja
+    - context:
+        smtp_ports: {{ smtp_ports }}
+    - user: root
+    - group: root
+    - mode: "0644"
+    - require:
+      - pkg: pmg-smtp-monitoring-packages
+
+pmg-smtp-monitoring-systemd-reload:
+  cmd.run:
+    - name: systemctl daemon-reload
+    - onchanges:
+      - file: pmg-smtp-monitoring-service
+      - file: pmg-smtp-monitoring-timer
+
+pmg-smtp-monitoring-blackbox-service:
+  service.running:
+    - name: {{ blackbox_service }}
+    - enable: true
+    - require:
+      - pkg: pmg-smtp-monitoring-packages
+    - watch:
+      - file: pmg-smtp-monitoring-blackbox-config
+
+pmg-smtp-monitoring-timer-running:
+  service.running:
+    - name: pmg-smtp-metrics.timer
+    - enable: true
+    - require:
+      - cmd: pmg-smtp-monitoring-systemd-reload
+      - file: pmg-smtp-monitoring-script
+
+pmg-smtp-monitoring-run-once:
+  cmd.run:
+    - name: /usr/local/bin/pmg-smtp-metrics.sh
+    - require:
+      - file: pmg-smtp-monitoring-script
+{% endif %}

pmg/verify.sls

@@ -13,3 +13,15 @@ pmg-verify-cluster:
 pmg-verify-sshd-config:
   cmd.run:
     - name: sshd -t
+
+pmg-verify-blackbox-service:
+  cmd.run:
+    - name: systemctl is-active --quiet prometheus-blackbox-exporter
+
+pmg-verify-smtp-metrics-timer:
+  cmd.run:
+    - name: systemctl is-active --quiet pmg-smtp-metrics.timer
+
+pmg-verify-smtp-metrics-exposed:
+  cmd.run:
+    - name: "curl -fsS http://127.0.0.1:9100/metrics | awk 'BEGIN { ok=0 } /^pmg_smtp_sent_recent / { ok=1 } END { exit(ok ? 0 : 1) }'"