diff --git a/README.md b/README.md index ce780cc..09e8fd1 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ - `pillar/pmg.sls` — переменные и параметры PMG. - `pmg/*.sls` — состояния: - `common` (базовые пакеты), + - `node_exporter` (установка и включение Prometheus node exporter), + - `smtp_monitoring` (SMTP-метрики: blackbox probe для 25/26 и postfix queue/connection gauges), - `ssh_hardening`, - `custom_check` (кастомная AV-проверка PMG через KESL), - `security_agents` (Wazuh/Qualys), @@ -50,6 +52,8 @@ make check MINION_ID=p-mailgw-x1-01.lmru.tech - `pmg.ssh_password_authentication` - `pmg.journald_system_max_use` - `pmg.journald_max_file_sec` +- `pmg.node_exporter.*` +- `pmg.smtp_monitoring.*` - `pmg.pmg_services` - `pmg.update_dist_upgrade` - `pmg.update_auto_reboot` @@ -60,6 +64,15 @@ make check MINION_ID=p-mailgw-x1-01.lmru.tech - `pmg.wazuh.*` - `pmg.qualys.*` +`pmg.node_exporter.args` по умолчанию включает textfile collector: +`--collector.textfile.directory=/var/lib/prometheus/node-exporter`. + +`pmg.smtp_monitoring` публикует расширенные SMTP метрики в node_exporter: +- queue total/deferred + age oldest message; +- SMTP listen/established для портов 25/26; +- recent window counters (`pmg.smtp_monitoring.log_window_seconds`) по логам postfix: + sent, received, deferred, bounced, rejected, tls failures, smtp connections. + ## Отдельное обновление PMG Перед обновлением: diff --git a/pillar/pmg.sls b/pillar/pmg.sls index 076bdb7..af646a9 100644 --- a/pillar/pmg.sls +++ b/pillar/pmg.sls @@ -12,6 +12,23 @@ pmg: journald_system_max_use: 1G journald_max_file_sec: 1month + node_exporter: + enabled: true + package: prometheus-node-exporter + service: prometheus-node-exporter + textfile_directory: /var/lib/prometheus/node-exporter + args: --collector.textfile.directory=/var/lib/prometheus/node-exporter + + smtp_monitoring: + enabled: true + log_window_seconds: 300 + ports: + - 25 + - 26 + blackbox: + package: prometheus-blackbox-exporter + service: prometheus-blackbox-exporter + pmg_services: - postfix - pmg-smtp-filter diff --git a/pmg/custom_check.sls b/pmg/custom_check.sls index 9023319..8b612f1 100644 --- a/pmg/custom_check.sls +++ b/pmg/custom_check.sls @@ -30,7 +30,7 @@ pmg-custom-check-ksc-superuser-sudoers: - group: root - mode: '0440' - contents: - - {{ ksc_superuser_name }} ALL = (ALL) NOPASSWD: ALL + - '{{ ksc_superuser_name }} ALL = (ALL) NOPASSWD: ALL' - check_cmd: visudo -cf - require: - pkg: pmg-custom-check-ksc-superuser-sudo-package diff --git a/pmg/files/blackbox.yml.jinja b/pmg/files/blackbox.yml.jinja new file mode 100644 index 0000000..6038bad --- /dev/null +++ b/pmg/files/blackbox.yml.jinja @@ -0,0 +1,12 @@ +modules: +{% for port in smtp_ports %} + smtp_{{ port }}: + prober: tcp + timeout: 5s + tcp: + preferred_ip_protocol: ip4 + query_response: + - expect: "^220[ -]" + - send: "EHLO blackbox.local" + - expect: "^250[ -]" +{% endfor %} diff --git a/pmg/files/pmg-smtp-metrics.service.jinja b/pmg/files/pmg-smtp-metrics.service.jinja new file mode 100644 index 0000000..398f2ec --- /dev/null +++ b/pmg/files/pmg-smtp-metrics.service.jinja @@ -0,0 +1,8 @@ +[Unit] +Description=Generate PMG SMTP metrics for node_exporter textfile collector +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/pmg-smtp-metrics.sh diff --git a/pmg/files/pmg-smtp-metrics.sh.jinja b/pmg/files/pmg-smtp-metrics.sh.jinja new file mode 100644 index 0000000..d4da827 --- /dev/null +++ b/pmg/files/pmg-smtp-metrics.sh.jinja @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -euo pipefail + +outfile="/var/lib/prometheus/node-exporter/pmg_smtp.prom" +tmpfile="$(mktemp)" +now_epoch="$(date +%s)" +window_seconds={{ smtp_log_window_seconds | int }} + +if [ "${window_seconds}" -lt 60 ]; then + window_seconds=60 +fi + +get_established_count() { + local port="$1" + ss -Htn state established "( sport = :${port} )" 2>/dev/null | awk 'END { print NR + 0 }' +} + +get_listen_up() { + local port="$1" + if ss -Hln "( sport = :${port} )" 2>/dev/null | awk 'END { exit(NR > 0 ? 0 : 1) }'; then + echo "1" + else + echo "0" + fi +} + +queue_total="$(postqueue -j 2>/dev/null | awk 'END { print NR + 0 }')" +queue_deferred="$(postqueue -j 2>/dev/null | awk -F'"' '/"queue_name":/ { if ($4 == "deferred") c++ } END { print c + 0 }')" +if command -v jq >/dev/null 2>&1; then + oldest_queue_epoch="$(postqueue -j 2>/dev/null | jq -sr 'map(.arrival_time // empty) | if length == 0 then 0 else min end' 2>/dev/null || echo "0")" +else + oldest_queue_epoch="0" +fi + +if [ "${oldest_queue_epoch}" -gt 0 ] 2>/dev/null; then + queue_oldest_age_seconds="$(( now_epoch - oldest_queue_epoch ))" + if [ "${queue_oldest_age_seconds}" -lt 0 ]; then + queue_oldest_age_seconds=0 + fi +else + queue_oldest_age_seconds=0 +fi + +log_slice="$(journalctl --no-pager -o cat --since "-${window_seconds} sec" 2>/dev/null || true)" +smtp_connections_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/smtpd.*connect from' || true)" +smtp_received_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)" +smtp_sent_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=sent' || true)" +smtp_deferred_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=deferred' || true)" +smtp_bounced_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=bounced' || true)" +smtp_rejected_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)" +smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)" + +{ + echo "# HELP pmg_postfix_queue_messages_total Number of messages in Postfix queue." + echo "# TYPE pmg_postfix_queue_messages_total gauge" + echo "pmg_postfix_queue_messages_total ${queue_total}" + echo "# HELP pmg_postfix_queue_deferred_messages Number of deferred messages in Postfix queue." + echo "# TYPE pmg_postfix_queue_deferred_messages gauge" + echo "pmg_postfix_queue_deferred_messages ${queue_deferred}" + echo "# HELP pmg_postfix_queue_oldest_age_seconds Age of oldest queued message in seconds." + echo "# TYPE pmg_postfix_queue_oldest_age_seconds gauge" + echo "pmg_postfix_queue_oldest_age_seconds ${queue_oldest_age_seconds}" + echo "# HELP pmg_smtp_metrics_window_seconds Time window in seconds used for recent SMTP counters." + echo "# TYPE pmg_smtp_metrics_window_seconds gauge" + echo "pmg_smtp_metrics_window_seconds ${window_seconds}" + echo "# HELP pmg_smtp_connections_recent Number of SMTP connections observed in recent window." + echo "# TYPE pmg_smtp_connections_recent gauge" + echo "pmg_smtp_connections_recent ${smtp_connections_recent}" + echo "# HELP pmg_smtp_received_recent Number of received messages observed in recent window." + echo "# TYPE pmg_smtp_received_recent gauge" + echo "pmg_smtp_received_recent ${smtp_received_recent}" + echo "# HELP pmg_smtp_sent_recent Number of sent messages observed in recent window." + echo "# TYPE pmg_smtp_sent_recent gauge" + echo "pmg_smtp_sent_recent ${smtp_sent_recent}" + echo "# HELP pmg_smtp_deferred_recent Number of deferred deliveries observed in recent window." + echo "# TYPE pmg_smtp_deferred_recent gauge" + echo "pmg_smtp_deferred_recent ${smtp_deferred_recent}" + echo "# HELP pmg_smtp_bounced_recent Number of bounced deliveries observed in recent window." + echo "# TYPE pmg_smtp_bounced_recent gauge" + echo "pmg_smtp_bounced_recent ${smtp_bounced_recent}" + echo "# HELP pmg_smtp_rejected_recent Number of rejected SMTP attempts observed in recent window." + echo "# TYPE pmg_smtp_rejected_recent gauge" + echo "pmg_smtp_rejected_recent ${smtp_rejected_recent}" + echo "# HELP pmg_smtp_tls_failures_recent Number of TLS/STARTTLS failures observed in recent window." + echo "# TYPE pmg_smtp_tls_failures_recent gauge" + echo "pmg_smtp_tls_failures_recent ${smtp_tls_fail_recent}" + echo "# HELP pmg_smtp_port_listen_up SMTP listen status on port (1 up, 0 down)." + echo "# TYPE pmg_smtp_port_listen_up gauge" + echo "# HELP pmg_smtp_port_established_connections Established SMTP connections on port." + echo "# TYPE pmg_smtp_port_established_connections gauge" +{% for port in smtp_ports %} + echo "pmg_smtp_port_listen_up{port=\"{{ port }}\"} $(get_listen_up {{ port }})" + echo "pmg_smtp_port_established_connections{port=\"{{ port }}\"} $(get_established_count {{ port }})" +{% endfor %} +} > "${tmpfile}" + +mv "${tmpfile}" "${outfile}" +chmod 0644 "${outfile}" diff --git a/pmg/files/pmg-smtp-metrics.timer.jinja b/pmg/files/pmg-smtp-metrics.timer.jinja new file mode 100644 index 0000000..265da39 --- /dev/null +++ b/pmg/files/pmg-smtp-metrics.timer.jinja @@ -0,0 +1,11 @@ +[Unit] +Description=Run PMG SMTP metrics exporter script every minute + +[Timer] +OnBootSec=30s +OnUnitActiveSec=1min +AccuracySec=15s +Unit=pmg-smtp-metrics.service + +[Install] +WantedBy=timers.target diff --git a/pmg/init.sls b/pmg/init.sls index da98408..4b16f86 100644 --- a/pmg/init.sls +++ b/pmg/init.sls @@ -1,5 +1,7 @@ include: - pmg.common + - pmg.node_exporter + - pmg.smtp_monitoring - pmg.ssh_hardening - pmg.custom_check - pmg.security_agents diff --git a/pmg/node_exporter.sls b/pmg/node_exporter.sls new file mode 100644 index 0000000..9c6c08e --- /dev/null +++ b/pmg/node_exporter.sls @@ -0,0 +1,39 @@ +{% set node_exporter_enabled = salt['pillar.get']('pmg:node_exporter:enabled', True) %} +{% set node_exporter_package = salt['pillar.get']('pmg:node_exporter:package', 'prometheus-node-exporter') %} +{% set node_exporter_service = salt['pillar.get']('pmg:node_exporter:service', 'prometheus-node-exporter') %} +{% set node_exporter_textfile_dir = salt['pillar.get']('pmg:node_exporter:textfile_directory', '/var/lib/prometheus/node-exporter') %} +{% set node_exporter_args = salt['pillar.get']('pmg:node_exporter:args', '--collector.textfile.directory=' ~ node_exporter_textfile_dir) %} + +{% if node_exporter_enabled %} +pmg-node-exporter-package: + pkg.installed: + - name: {{ node_exporter_package }} + +pmg-node-exporter-textfile-dir: + file.directory: + - name: {{ node_exporter_textfile_dir }} + - user: root + - group: root + - mode: "0755" + +pmg-node-exporter-defaults: + file.managed: + - name: /etc/default/prometheus-node-exporter + - user: root + - group: root + - mode: "0644" + - contents: + - "# Managed by Salt." + - "ARGS=\"{{ node_exporter_args }}\"" + +pmg-node-exporter-service: + service.running: + - name: {{ node_exporter_service }} + - enable: true + - require: + - pkg: pmg-node-exporter-package + - file: pmg-node-exporter-textfile-dir + - file: pmg-node-exporter-defaults + - watch: + - file: pmg-node-exporter-defaults +{% endif %} diff --git a/pmg/smtp_monitoring.sls b/pmg/smtp_monitoring.sls new file mode 100644 index 0000000..92adca7 --- /dev/null +++ b/pmg/smtp_monitoring.sls @@ -0,0 +1,94 @@ +{% set smtp_monitoring_enabled = salt['pillar.get']('pmg:smtp_monitoring:enabled', True) %} +{% set blackbox_package = salt['pillar.get']('pmg:smtp_monitoring:blackbox:package', 'prometheus-blackbox-exporter') %} +{% set blackbox_service = salt['pillar.get']('pmg:smtp_monitoring:blackbox:service', 'prometheus-blackbox-exporter') %} +{% set smtp_ports = salt['pillar.get']('pmg:smtp_monitoring:ports', [25, 26]) %} +{% set smtp_log_window_seconds = salt['pillar.get']('pmg:smtp_monitoring:log_window_seconds', 300) %} + +{% if smtp_monitoring_enabled %} +pmg-smtp-monitoring-packages: + pkg.installed: + - pkgs: + - {{ blackbox_package }} + - iproute2 + - jq + +pmg-smtp-monitoring-metrics-dir: + file.directory: + - name: /var/lib/prometheus/node-exporter + - user: root + - group: root + - mode: "0755" + +pmg-smtp-monitoring-script: + file.managed: + - name: /usr/local/bin/pmg-smtp-metrics.sh + - source: salt://pmg/files/pmg-smtp-metrics.sh.jinja + - template: jinja + - context: + smtp_ports: {{ smtp_ports }} + smtp_log_window_seconds: {{ smtp_log_window_seconds }} + - user: root + - group: root + - mode: "0755" + - require: + - file: pmg-smtp-monitoring-metrics-dir + +pmg-smtp-monitoring-service: + file.managed: + - name: /etc/systemd/system/pmg-smtp-metrics.service + - source: salt://pmg/files/pmg-smtp-metrics.service.jinja + - user: root + - group: root + - mode: "0644" + +pmg-smtp-monitoring-timer: + file.managed: + - name: /etc/systemd/system/pmg-smtp-metrics.timer + - source: salt://pmg/files/pmg-smtp-metrics.timer.jinja + - user: root + - group: root + - mode: "0644" + +pmg-smtp-monitoring-blackbox-config: + file.managed: + - name: /etc/prometheus/blackbox.yml + - source: salt://pmg/files/blackbox.yml.jinja + - template: jinja + - context: + smtp_ports: {{ smtp_ports }} + - user: root + - group: root + - mode: "0644" + - require: + - pkg: pmg-smtp-monitoring-packages + +pmg-smtp-monitoring-systemd-reload: + cmd.run: + - name: systemctl daemon-reload + - onchanges: + - file: pmg-smtp-monitoring-service + - file: pmg-smtp-monitoring-timer + +pmg-smtp-monitoring-blackbox-service: + service.running: + - name: {{ blackbox_service }} + - enable: true + - require: + - pkg: pmg-smtp-monitoring-packages + - watch: + - file: pmg-smtp-monitoring-blackbox-config + +pmg-smtp-monitoring-timer-running: + service.running: + - name: pmg-smtp-metrics.timer + - enable: true + - require: + - cmd: pmg-smtp-monitoring-systemd-reload + - file: pmg-smtp-monitoring-script + +pmg-smtp-monitoring-run-once: + cmd.run: + - name: /usr/local/bin/pmg-smtp-metrics.sh + - require: + - file: pmg-smtp-monitoring-script +{% endif %} diff --git a/pmg/verify.sls b/pmg/verify.sls index ebb9595..abc7ab9 100644 --- a/pmg/verify.sls +++ b/pmg/verify.sls @@ -13,3 +13,15 @@ pmg-verify-cluster: pmg-verify-sshd-config: cmd.run: - name: sshd -t + +pmg-verify-blackbox-service: + cmd.run: + - name: systemctl is-active --quiet prometheus-blackbox-exporter + +pmg-verify-smtp-metrics-timer: + cmd.run: + - name: systemctl is-active --quiet pmg-smtp-metrics.timer + +pmg-verify-smtp-metrics-exposed: + cmd.run: + - name: "curl -fsS http://127.0.0.1:9100/metrics | awk 'BEGIN { ok=0 } /^pmg_smtp_sent_recent / { ok=1 } END { exit(ok ? 0 : 1) }'"