Align Postfix identity with system hostname on PMG nodes.

Add an idempotent baseline step that enforces myhostname/mydomain from hostname -f and reloads postfix only when drift is detected.

Co-authored-by: Cursor <cursoragent@cursor.com>

Makefile

@@ -7,8 +7,10 @@ YAMLLINT ?= yamllint
 STATE_ROOT ?= .
 PILLAR_ROOT ?= ./pillar
 MINION_ID ?= p-mailgw-x2-01.lmru.tech
+MASTER_IP ?= 10.80.38.152
+MASTER_HOST ?= p-mailgw-x1-01.lmru.tech
 
-.PHONY: lint check apply verify update-check update
+.PHONY: lint check apply verify update-check update cluster-bootstrap
 
 lint:
 >@if command -v $(SALT_LINT) >/dev/null 2>&1; then \
@@ -36,3 +38,6 @@ update-check:
 
 update:
 >sudo $(SALT_CALL) --local --id $(MINION_ID) --file-root=$(STATE_ROOT) --pillar-root=$(PILLAR_ROOT) state.apply pmg.update
+
+cluster-bootstrap:
+>bash "./scripts/cluster-bootstrap.sh" "$(MINION_ID)" "$(MASTER_HOST)" "$(MASTER_IP)"

README.md

@@ -8,6 +8,8 @@
 - `pillar/pmg.sls` — переменные и параметры PMG.
 - `pmg/*.sls` — состояния:
   - `common` (базовые пакеты),
+  - `node_exporter` (установка и включение Prometheus node exporter),
+  - `smtp_monitoring` (SMTP-метрики: blackbox probe для 25/26 и postfix queue/connection gauges),
   - `ssh_hardening`,
   - `custom_check` (кастомная AV-проверка PMG через KESL),
   - `security_agents` (Wazuh/Qualys),
@@ -50,16 +52,53 @@ make check MINION_ID=p-mailgw-x1-01.lmru.tech
 - `pmg.ssh_password_authentication`
 - `pmg.journald_system_max_use`
 - `pmg.journald_max_file_sec`
+- `pmg.node_exporter.*`
+- `pmg.smtp_monitoring.*`
 - `pmg.pmg_services`
+- `pmg.pmg_cluster_only_services`
+- `pmg.cluster_join.*`
 - `pmg.update_dist_upgrade`
 - `pmg.update_auto_reboot`
 - `pmg.update_reboot_delay_minutes`
 - `pmg.custom_check.*`
+  - `pmg.custom_check.log_dir_user|log_dir_group|log_dir_mode|log_file_mode`
 - `pmg.manage_hosts_file`
 - `pmg.hosts_file_entries`
 - `pmg.wazuh.*`
+- `pmg.kesl.*`
+- `pmg.klnagent.*`
 - `pmg.qualys.*`
 
+`pmg.node_exporter.args` по умолчанию включает textfile collector:
+`--collector.textfile.directory=/var/lib/prometheus/node-exporter`.
+
+`pmg.smtp_monitoring` публикует расширенные SMTP метрики в node_exporter:
+- queue total/deferred + age oldest message;
+- SMTP listen/established для портов 25/26;
+- recent window counters (`pmg.smtp_monitoring.log_window_seconds`) по логам postfix:
+  sent, received, deferred, bounced, rejected, tls failures, smtp connections.
+
+`pmg.cluster_join` делает:
+- precheck passwordless SSH с новой ноды на `pmg.cluster_join.master_ip`;
+- non-interactive `pmgcm join` при заданном `pmg.cluster_join.password`.
+
+Важно: `pmgcm join` использует пароль PMG (`root@pam` на master) для получения ticket
+(это не SSH-пароль). Без `pmg.cluster_join.password` state делает skip-уведомление.
+
+Рекомендуется передавать пароль через переменную окружения (без сохранения в `pillar/pmg.sls`):
+
+```bash
+export PMG_CLUSTER_JOIN_PASSWORD='***'
+PILLAR_B64="$(python3 -c 'import json, os, base64; print(base64.b64encode(json.dumps({\"pmg\": {\"cluster_join\": {\"password\": os.environ[\"PMG_CLUSTER_JOIN_PASSWORD\"]}}}).encode()).decode())')"
+ssh root@p-mailgw-x1-02.lmru.tech "cd /root/mailgw-salt && salt-call --local --id p-mailgw-x1-02.lmru.tech --file-root=. --pillar-root=./pillar state.apply pmg.cluster_join pillar=\"\$(python3 -c \"import base64; print(base64.b64decode(\\\"${PILLAR_B64}\\\").decode())\")\""
+```
+
+Для полностью автоматического bootstrap SSH-доверия (без паролей) с управляющей машины:
+
+```bash
+make cluster-bootstrap MINION_ID=p-mailgw-x1-02.lmru.tech MASTER_HOST=p-mailgw-x1-01.lmru.tech MASTER_IP=10.80.38.152
+```
+
 ## Отдельное обновление PMG
 
 Перед обновлением:
@@ -88,4 +127,17 @@ State `pmg.update` выполняет:
 Коротко про `pmg.custom_check`:
 - готовит окружение (`pwgen`, `/var/log/mail_av`, `/usr/local/bin/pmg-custom-check`);
 - готовит Kaspersky-учетку (`pmg.custom_check.ksc_superuser.*`; сейчас `superuser` + `NOPASSWD`);
+- настраивает исключения KESL из сканирования через `pmg.custom_check.kesl_excluded_paths`
+  (по умолчанию: `/var/spool/pmg`, `/var/lib/clamav`, `/tmp/.proxdump_*`) и делает
+  `systemctl restart kesl` только при изменении этих настроек;
 - если `kesl/klnagent64` установлены — включает сервисы и `custom_check` в PMG, если нет — выводит уведомление "ставится через Kaspersky Administration Center"; сервисы включатся вторым прогоном после установки пакетов.
+
+`pmg.security_agents` дополнительно ставит/настраивает `kesl` и `klnagent64`:
+- для `kesl`:
+  - скачивает `.deb` из `pmg.kesl.package_url`;
+  - формирует `autoinstall.ini` и запускает `/opt/kaspersky/kesl/bin/kesl-setup.pl --autoinstall=...`;
+  - поднимает сервис `kesl`;
+- скачивает `.deb` из `pmg.klnagent.package_url`;
+- формирует файл ответов `KLAUTOANSWERS` (минимум: `KLNAGENT_SERVER`,
+  `KLNAGENT_AUTOINSTALL`, `EULA_ACCEPTED`);
+- устанавливает/переустанавливает пакет в тихом режиме и поднимает сервис `klnagent64`.

pillar/pmg.sls

@@ -12,6 +12,23 @@ pmg:
   journald_system_max_use: 1G
   journald_max_file_sec: 1month
 
+  node_exporter:
+    enabled: true
+    package: prometheus-node-exporter
+    service: prometheus-node-exporter
+    textfile_directory: /var/lib/prometheus/node-exporter
+    args: --collector.textfile.directory=/var/lib/prometheus/node-exporter
+
+  smtp_monitoring:
+    enabled: true
+    log_window_seconds: 300
+    ports:
+      - 25
+      - 26
+    blackbox:
+      package: prometheus-blackbox-exporter
+      service: prometheus-blackbox-exporter
+
   pmg_services:
     - postfix
     - pmg-smtp-filter
@@ -19,6 +36,14 @@ pmg:
     - pmgdaemon
     - pmgmirror
     - pmgtunnel
+  pmg_cluster_only_services:
+    - pmgmirror
+    - pmgtunnel
+  cluster_join:
+    enabled: true
+    master_ip: 10.80.38.152
+    fingerprint: "4C:63:ED:BB:01:F0:63:5B:92:AC:37:D8:52:D9:0A:CB:5B:78:EE:3A:36:83:86:29:C8:19:D2:EC:CE:AC:88:EC"
+    password: ""
   update_dist_upgrade: true
   update_auto_reboot: false
   update_reboot_delay_minutes: 1
@@ -27,6 +52,10 @@ pmg:
     enabled: true
     script_path: /usr/local/bin/pmg-custom-check
     log_dir: /var/log/mail_av
+    log_dir_user: pmg-smtp-filter
+    log_dir_group: pmg
+    log_dir_mode: "0770"
+    log_file_mode: "0660"
     managed_packages:
       - pwgen
     ksc_superuser:
@@ -36,6 +65,10 @@ pmg:
     required_services:
       - kesl
       - klnagent64
+    kesl_excluded_paths:
+      - /var/spool/pmg
+      - /var/lib/clamav
+      - /tmp/.proxdump_*
 
   manage_hosts_file: false
   hosts_file_entries:
@@ -43,10 +76,18 @@ pmg:
       hostname: p-mailgw-x1-01
       aliases:
         - p-mailgw-x1-01.lmru.tech
+    - ip: 10.80.38.153
+      hostname: p-mailgw-x1-02
+      aliases:
+        - p-mailgw-x1-02.lmru.tech
     - ip: 10.220.41.152
       hostname: p-mailgw-x2-01
       aliases:
         - p-mailgw-x2-01.lmru.tech
+    - ip: 10.220.41.153
+      hostname: p-mailgw-x2-02
+      aliases:
+        - p-mailgw-x2-02.lmru.tech
 
   wazuh:
     enabled: true
@@ -54,6 +95,31 @@ pmg:
     manager: 10.80.120.205
     group: test_auditd_unix
 
+  kesl:
+    enabled: true
+    package_url: https://products.s.kaspersky-labs.com/endpoints/keslinux10/12.4.0.1225/multilanguage-INT-12.4.0.1225/167a2f9c5bdb4e869f4603fbc3a829c1/kesl_12.4.0-1225_amd64.deb
+    package_path: /opt/kesl_12.4.0-1225_amd64.deb
+    autoinstall_path: /etc/opt/kaspersky/kesl/autoinstall.ini
+    ksvla_mode: "No"
+    endpoint_agent_mode: "No"
+    eula_agreed: "Yes"
+    privacy_policy_agreed: "Yes"
+    use_ksn: "No"
+    group_clean: "No"
+    locale: ""
+    install_license: ""
+    updater_source: SCServer
+    update_execute: "No"
+
+  klnagent:
+    enabled: true
+    package_url: https://products.s.kaspersky-labs.com/administrationkit/ksc10/16.2.0.1023/russian-INT-29664057-ru/2129c97dba254c9fb8a4161f29a2241c/klnagent64_16.2.0-1023_amd64.deb
+    package_path: /opt/klnagent64_16.2.0-1023_amd64.deb
+    administration_server: 10.80.38.76
+    autoanswers_path: /etc/opt/kaspersky/klnagent64/autoanswers.conf
+    autoinstall: true
+    eula_accepted: true
+
   qualys:
     enabled: true
     package_url: https://storage.yandexcloud.net/qualys-agent/QualysCloudAgent.deb

pmg/cluster_join.sls

@@ -0,0 +1,30 @@
+{% set cluster_join_enabled = salt['pillar.get']('pmg:cluster_join:enabled', False) %}
+{% set cluster_join_master_ip = salt['pillar.get']('pmg:cluster_join:master_ip', '') %}
+{% set cluster_join_fingerprint = salt['pillar.get']('pmg:cluster_join:fingerprint', '') %}
+{% set cluster_join_password = salt['pillar.get']('pmg:cluster_join:password', '') %}
+
+{% if cluster_join_enabled and cluster_join_master_ip and cluster_join_fingerprint %}
+pmg-cluster-join-passwordless-ssh-check:
+  cmd.run:
+    - name: ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new root@{{ cluster_join_master_ip }} "exit 0"
+    - timeout: 20
+
+{% if cluster_join_password %}
+pmg-cluster-join:
+  cmd.run:
+    - name: pmgcm join {{ cluster_join_master_ip }} --fingerprint {{ cluster_join_fingerprint }}
+    - stdin: "{{ cluster_join_password }}"
+    - timeout: 60
+    - require:
+      - cmd: pmg-cluster-join-passwordless-ssh-check
+    - unless: "pmgcm status 2>/dev/null | awk 'BEGIN { ok=1 } /no cluster defined/ { ok=0 } END { exit(ok ? 0 : 1) }'"
+{% else %}
+pmg-cluster-join-skipped-password:
+  test.show_notification:
+    - text: "Cluster join skipped: set pmg.cluster_join.password (PMG root@pam password on master) for non-interactive join."
+{% endif %}
+{% elif cluster_join_enabled %}
+pmg-cluster-join-skipped:
+  test.show_notification:
+    - text: "Cluster join skipped: set pmg.cluster_join.master_ip and pmg.cluster_join.fingerprint in pillar."
+{% endif %}

pmg/custom_check.sls

@@ -1,8 +1,13 @@
 {% set custom_check_enabled = salt['pillar.get']('pmg:custom_check:enabled', True) %}
 {% set custom_check_script_path = salt['pillar.get']('pmg:custom_check:script_path', '/usr/local/bin/pmg-custom-check') %}
 {% set custom_check_log_dir = salt['pillar.get']('pmg:custom_check:log_dir', '/var/log/mail_av') %}
+{% set custom_check_log_dir_user = salt['pillar.get']('pmg:custom_check:log_dir_user', 'pmg-smtp-filter') %}
+{% set custom_check_log_dir_group = salt['pillar.get']('pmg:custom_check:log_dir_group', 'pmg') %}
+{% set custom_check_log_dir_mode = salt['pillar.get']('pmg:custom_check:log_dir_mode', '0770') %}
+{% set custom_check_log_file_mode = salt['pillar.get']('pmg:custom_check:log_file_mode', '0660') %}
 {% set custom_check_managed_packages = salt['pillar.get']('pmg:custom_check:managed_packages', ['pwgen']) %}
 {% set custom_check_required_services = salt['pillar.get']('pmg:custom_check:required_services', ['kesl', 'klnagent64']) %}
+{% set custom_check_kesl_excluded_paths = salt['pillar.get']('pmg:custom_check:kesl_excluded_paths', ['/var/spool/pmg', '/var/lib/clamav', '/tmp/.proxdump_*']) %}
 {% set ksc_superuser_enabled = salt['pillar.get']('pmg:custom_check:ksc_superuser:enabled', False) %}
 {% set ksc_superuser_name = salt['pillar.get']('pmg:custom_check:ksc_superuser:name', '') %}
 {% set ksc_superuser_manage_sudoers = salt['pillar.get']('pmg:custom_check:ksc_superuser:manage_sudoers', True) %}
@@ -30,7 +35,7 @@ pmg-custom-check-ksc-superuser-sudoers:
     - group: root
     - mode: '0440'
     - contents:
-      - {{ ksc_superuser_name }} ALL = (ALL) NOPASSWD: ALL
+      - '{{ ksc_superuser_name }} ALL = (ALL) NOPASSWD: ALL'
     - check_cmd: visudo -cf
     - require:
       - pkg: pmg-custom-check-ksc-superuser-sudo-package
@@ -49,13 +54,20 @@ pmg-custom-check-managed-packages:
 pmg-custom-check-log-dir:
   file.directory:
     - name: {{ custom_check_log_dir }}
-    - user: root
+    - user: {{ custom_check_log_dir_user }}
-    - group: root
+    - group: {{ custom_check_log_dir_group }}
-    - mode: '0755'
+    - mode: '{{ custom_check_log_dir_mode }}'
+    - recurse:
+      - user
+      - group
 
 pmg-custom-check-log-file:
-  file.touch:
+  file.managed:
     - name: {{ custom_check_log_dir }}/checked
+    - user: {{ custom_check_log_dir_user }}
+    - group: {{ custom_check_log_dir_group }}
+    - mode: '{{ custom_check_log_file_mode }}'
+    - replace: false
     - require:
       - file: pmg-custom-check-log-dir
 
@@ -93,6 +105,34 @@ pmg-custom-check-service-{{ service_name }}:
       - test: pmg-custom-check-kaspersky-ready
 {% endfor %}
 
+{% for excluded_path in custom_check_kesl_excluded_paths %}
+pmg-custom-check-kesl-excluded-scope-{{ loop.index0 }}:
+  cmd.run:
+    - name: >-
+        kesl-control --set-settings File_Threat_Protection
+        ExcludedFromScanScope.item_{{ "%04d" | format(loop.index0) }}.Path='{{ excluded_path }}'
+        ExcludedFromScanScope.item_{{ "%04d" | format(loop.index0) }}.UseScanArea=Yes
+        ExcludedFromScanScope.item_{{ "%04d" | format(loop.index0) }}.AreaMask.item_0000='*'
+    - unless: >-
+        kesl-control --get-settings File_Threat_Protection 2>/dev/null
+        | grep -Fq "Path={{ excluded_path }}"
+    - require:
+      - test: pmg-custom-check-kaspersky-ready
+{% for service_name in custom_check_required_services %}
+      - service: pmg-custom-check-service-{{ service_name }}
+{% endfor %}
+{% endfor %}
+
+pmg-custom-check-kesl-restart:
+  cmd.wait:
+    - name: systemctl restart kesl
+    - watch:
+{% for excluded_path in custom_check_kesl_excluded_paths %}
+      - cmd: pmg-custom-check-kesl-excluded-scope-{{ loop.index0 }}
+{% endfor %}
+    - require:
+      - test: pmg-custom-check-kaspersky-ready
+
 pmg-custom-check-enable-in-pmg:
   cmd.run:
     - name: pmgsh set /config/admin --custom_check 1

pmg/files/blackbox.yml.jinja

@@ -0,0 +1,12 @@
+modules:
+{% for port in smtp_ports %}
+  smtp_{{ port }}:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      preferred_ip_protocol: ip4
+      query_response:
+        - expect: "^220[ -]"
+        - send: "EHLO blackbox.local"
+        - expect: "^250[ -]"
+{% endfor %}

pmg/files/pmg-smtp-metrics.service.jinja

@@ -0,0 +1,8 @@
+[Unit]
+Description=Generate PMG SMTP metrics for node_exporter textfile collector
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/pmg-smtp-metrics.sh

pmg/files/pmg-smtp-metrics.sh.jinja

@@ -0,0 +1,219 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+outfile="/var/lib/prometheus/node-exporter/pmg_smtp.prom"
+tmpfile="$(mktemp)"
+statefile="/var/lib/prometheus/node-exporter/pmg_smtp.state"
+statetmp="$(mktemp)"
+now_epoch="$(date +%s)"
+window_seconds={{ smtp_log_window_seconds | int }}
+
+if [ "${window_seconds}" -lt 60 ]; then
+  window_seconds=60
+fi
+
+to_int() {
+  local value="${1:-0}"
+  if [[ "${value}" =~ ^[0-9]+$ ]]; then
+    echo "${value}"
+  else
+    echo "0"
+  fi
+}
+
+get_established_count() {
+  local port="$1"
+  ss -Htn state established "( sport = :${port} )" 2>/dev/null | awk 'END { print NR + 0 }'
+}
+
+get_listen_up() {
+  local port="$1"
+  if ss -Hln "( sport = :${port} )" 2>/dev/null | awk 'END { exit(NR > 0 ? 0 : 1) }'; then
+    echo "1"
+  else
+    echo "0"
+  fi
+}
+
+service_up() {
+  local service="$1"
+  if systemctl is-active --quiet "${service}" 2>/dev/null; then
+    echo "1"
+  else
+    echo "0"
+  fi
+}
+
+last_cursor=""
+total_smtp_connections=0
+total_smtp_received=0
+total_smtp_sent=0
+total_smtp_deferred=0
+total_smtp_bounced=0
+total_smtp_rejected=0
+total_smtp_tls_failures=0
+prev_queue_total=0
+prev_queue_ts=0
+
+if [ -f "${statefile}" ]; then
+  while IFS='=' read -r key value; do
+    case "${key}" in
+      last_cursor) last_cursor="${value}" ;;
+      total_smtp_connections) total_smtp_connections="$(to_int "${value}")" ;;
+      total_smtp_received) total_smtp_received="$(to_int "${value}")" ;;
+      total_smtp_sent) total_smtp_sent="$(to_int "${value}")" ;;
+      total_smtp_deferred) total_smtp_deferred="$(to_int "${value}")" ;;
+      total_smtp_bounced) total_smtp_bounced="$(to_int "${value}")" ;;
+      total_smtp_rejected) total_smtp_rejected="$(to_int "${value}")" ;;
+      total_smtp_tls_failures) total_smtp_tls_failures="$(to_int "${value}")" ;;
+      prev_queue_total) prev_queue_total="$(to_int "${value}")" ;;
+      prev_queue_ts) prev_queue_ts="$(to_int "${value}")" ;;
+    esac
+  done < "${statefile}"
+fi
+
+queue_total="$(postqueue -j 2>/dev/null | awk 'END { print NR + 0 }')"
+queue_deferred="$(postqueue -j 2>/dev/null | awk -F'"' '/"queue_name":/ { if ($4 == "deferred") c++ } END { print c + 0 }')"
+if command -v jq >/dev/null 2>&1; then
+  oldest_queue_epoch="$(postqueue -j 2>/dev/null | jq -sr 'map(.arrival_time // empty) | if length == 0 then 0 else min end' 2>/dev/null || echo "0")"
+else
+  oldest_queue_epoch="0"
+fi
+
+if [ "${oldest_queue_epoch}" -gt 0 ] 2>/dev/null; then
+  queue_oldest_age_seconds="$(( now_epoch - oldest_queue_epoch ))"
+  if [ "${queue_oldest_age_seconds}" -lt 0 ]; then
+    queue_oldest_age_seconds=0
+  fi
+else
+  queue_oldest_age_seconds=0
+fi
+
+if [ "${prev_queue_ts}" -gt 0 ] && [ "${now_epoch}" -gt "${prev_queue_ts}" ]; then
+  queue_delta_seconds="$(( now_epoch - prev_queue_ts ))"
+  queue_drain_per_minute="$(awk -v prev="${prev_queue_total}" -v curr="${queue_total}" -v dt="${queue_delta_seconds}" 'BEGIN { d = prev - curr; if (dt <= 0 || d <= 0) print 0; else printf "%.3f", (d * 60.0) / dt }')"
+else
+  queue_drain_per_minute="0"
+fi
+
+log_slice="$(journalctl --no-pager -o cat --since "-${window_seconds} sec" 2>/dev/null || true)"
+smtp_connections_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/smtpd.*connect from' || true)"
+smtp_received_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)"
+smtp_sent_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=sent' || true)"
+smtp_deferred_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=deferred' || true)"
+smtp_bounced_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'status=bounced' || true)"
+smtp_rejected_recent="$(printf '%s\n' "${log_slice}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)"
+smtp_tls_fail_recent="$(printf '%s\n' "${log_slice}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)"
+
+journal_cursor_args=()
+if [ -n "${last_cursor}" ]; then
+  journal_cursor_args+=(--after-cursor "${last_cursor}")
+fi
+journal_delta="$(journalctl --no-pager -o cat "${journal_cursor_args[@]}" 2>/dev/null || true)"
+delta_smtp_connections="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/smtpd.*connect from' || true)"
+delta_smtp_received="$(printf '%s\n' "${journal_delta}" | grep -Ec 'postfix/qmgr.*: .*: from=<' || true)"
+delta_smtp_sent="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=sent' || true)"
+delta_smtp_deferred="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=deferred' || true)"
+delta_smtp_bounced="$(printf '%s\n' "${journal_delta}" | grep -Ec 'status=bounced' || true)"
+delta_smtp_rejected="$(printf '%s\n' "${journal_delta}" | grep -Ec 'NOQUEUE: reject:| reject:' || true)"
+delta_smtp_tls_failures="$(printf '%s\n' "${journal_delta}" | grep -Eci '(tls|starttls).*(fail|error|warning|lost)' || true)"
+
+total_smtp_connections="$(( total_smtp_connections + delta_smtp_connections ))"
+total_smtp_received="$(( total_smtp_received + delta_smtp_received ))"
+total_smtp_sent="$(( total_smtp_sent + delta_smtp_sent ))"
+total_smtp_deferred="$(( total_smtp_deferred + delta_smtp_deferred ))"
+total_smtp_bounced="$(( total_smtp_bounced + delta_smtp_bounced ))"
+total_smtp_rejected="$(( total_smtp_rejected + delta_smtp_rejected ))"
+total_smtp_tls_failures="$(( total_smtp_tls_failures + delta_smtp_tls_failures ))"
+
+current_cursor="$(journalctl --no-pager -n 1 -o cat --show-cursor 2>/dev/null | awk -F': ' '/^-- cursor: / { c = $2 } END { print c }')"
+if [ -z "${current_cursor}" ]; then
+  current_cursor="${last_cursor}"
+fi
+
+{
+  echo "# HELP pmg_postfix_queue_messages_total Number of messages in Postfix queue."
+  echo "# TYPE pmg_postfix_queue_messages_total gauge"
+  echo "pmg_postfix_queue_messages_total ${queue_total}"
+  echo "# HELP pmg_postfix_queue_deferred_messages Number of deferred messages in Postfix queue."
+  echo "# TYPE pmg_postfix_queue_deferred_messages gauge"
+  echo "pmg_postfix_queue_deferred_messages ${queue_deferred}"
+  echo "# HELP pmg_postfix_queue_oldest_age_seconds Age of oldest queued message in seconds."
+  echo "# TYPE pmg_postfix_queue_oldest_age_seconds gauge"
+  echo "pmg_postfix_queue_oldest_age_seconds ${queue_oldest_age_seconds}"
+  echo "# HELP pmg_postfix_queue_drain_per_minute Queue drain rate in messages per minute (only positive drain)."
+  echo "# TYPE pmg_postfix_queue_drain_per_minute gauge"
+  echo "pmg_postfix_queue_drain_per_minute ${queue_drain_per_minute}"
+  echo "# HELP pmg_smtp_metrics_window_seconds Time window in seconds used for recent SMTP counters."
+  echo "# TYPE pmg_smtp_metrics_window_seconds gauge"
+  echo "pmg_smtp_metrics_window_seconds ${window_seconds}"
+  echo "# HELP pmg_smtp_connections_recent Number of SMTP connections observed in recent window."
+  echo "# TYPE pmg_smtp_connections_recent gauge"
+  echo "pmg_smtp_connections_recent ${smtp_connections_recent}"
+  echo "# HELP pmg_smtp_received_recent Number of received messages observed in recent window."
+  echo "# TYPE pmg_smtp_received_recent gauge"
+  echo "pmg_smtp_received_recent ${smtp_received_recent}"
+  echo "# HELP pmg_smtp_sent_recent Number of sent messages observed in recent window."
+  echo "# TYPE pmg_smtp_sent_recent gauge"
+  echo "pmg_smtp_sent_recent ${smtp_sent_recent}"
+  echo "# HELP pmg_smtp_deferred_recent Number of deferred deliveries observed in recent window."
+  echo "# TYPE pmg_smtp_deferred_recent gauge"
+  echo "pmg_smtp_deferred_recent ${smtp_deferred_recent}"
+  echo "# HELP pmg_smtp_bounced_recent Number of bounced deliveries observed in recent window."
+  echo "# TYPE pmg_smtp_bounced_recent gauge"
+  echo "pmg_smtp_bounced_recent ${smtp_bounced_recent}"
+  echo "# HELP pmg_smtp_rejected_recent Number of rejected SMTP attempts observed in recent window."
+  echo "# TYPE pmg_smtp_rejected_recent gauge"
+  echo "pmg_smtp_rejected_recent ${smtp_rejected_recent}"
+  echo "# HELP pmg_smtp_tls_failures_recent Number of TLS/STARTTLS failures observed in recent window."
+  echo "# TYPE pmg_smtp_tls_failures_recent gauge"
+  echo "pmg_smtp_tls_failures_recent ${smtp_tls_fail_recent}"
+  echo "# HELP pmg_smtp_connections_total Total SMTP connections seen in logs since metrics script state start."
+  echo "# TYPE pmg_smtp_connections_total counter"
+  echo "pmg_smtp_connections_total ${total_smtp_connections}"
+  echo "# HELP pmg_smtp_messages_total Total SMTP message events by direction/status since metrics script state start."
+  echo "# TYPE pmg_smtp_messages_total counter"
+  echo "pmg_smtp_messages_total{direction=\"received\",status=\"accepted\"} ${total_smtp_received}"
+  echo "pmg_smtp_messages_total{direction=\"sent\",status=\"sent\"} ${total_smtp_sent}"
+  echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"deferred\"} ${total_smtp_deferred}"
+  echo "pmg_smtp_messages_total{direction=\"delivery\",status=\"bounced\"} ${total_smtp_bounced}"
+  echo "pmg_smtp_messages_total{direction=\"inbound\",status=\"rejected\"} ${total_smtp_rejected}"
+  echo "# HELP pmg_smtp_tls_failures_total Total TLS/STARTTLS failures seen in logs since metrics script state start."
+  echo "# TYPE pmg_smtp_tls_failures_total counter"
+  echo "pmg_smtp_tls_failures_total ${total_smtp_tls_failures}"
+  echo "# HELP pmg_smtp_port_listen_up SMTP listen status on port (1 up, 0 down)."
+  echo "# TYPE pmg_smtp_port_listen_up gauge"
+  echo "# HELP pmg_smtp_port_established_connections Established SMTP connections on port."
+  echo "# TYPE pmg_smtp_port_established_connections gauge"
+  echo "# HELP pmg_service_up PMG-related service status (1 up, 0 down)."
+  echo "# TYPE pmg_service_up gauge"
+{% for port in smtp_ports %}
+  echo "pmg_smtp_port_listen_up{port=\"{{ port }}\"} $(get_listen_up {{ port }})"
+  echo "pmg_smtp_port_established_connections{port=\"{{ port }}\"} $(get_established_count {{ port }})"
+{% endfor %}
+  echo "pmg_service_up{service=\"postfix\"} $(service_up postfix)"
+  echo "pmg_service_up{service=\"pmg-smtp-filter\"} $(service_up pmg-smtp-filter)"
+  echo "pmg_service_up{service=\"pmgpolicy\"} $(service_up pmgpolicy)"
+  echo "pmg_service_up{service=\"pmgdaemon\"} $(service_up pmgdaemon)"
+  echo "pmg_service_up{service=\"pmgmirror\"} $(service_up pmgmirror)"
+  echo "pmg_service_up{service=\"pmgtunnel\"} $(service_up pmgtunnel)"
+} > "${tmpfile}"
+
+mv "${tmpfile}" "${outfile}"
+chmod 0644 "${outfile}"
+
+{
+  echo "last_cursor=${current_cursor}"
+  echo "total_smtp_connections=${total_smtp_connections}"
+  echo "total_smtp_received=${total_smtp_received}"
+  echo "total_smtp_sent=${total_smtp_sent}"
+  echo "total_smtp_deferred=${total_smtp_deferred}"
+  echo "total_smtp_bounced=${total_smtp_bounced}"
+  echo "total_smtp_rejected=${total_smtp_rejected}"
+  echo "total_smtp_tls_failures=${total_smtp_tls_failures}"
+  echo "prev_queue_total=${queue_total}"
+  echo "prev_queue_ts=${now_epoch}"
+} > "${statetmp}"
+
+mv "${statetmp}" "${statefile}"
+chmod 0644 "${statefile}"

pmg/files/pmg-smtp-metrics.timer.jinja

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run PMG SMTP metrics exporter script every minute
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec=1min
+AccuracySec=15s
+Unit=pmg-smtp-metrics.service
+
+[Install]
+WantedBy=timers.target

pmg/init.sls

@@ -1,8 +1,11 @@
 include:
   - pmg.common
+  - pmg.node_exporter
+  - pmg.smtp_monitoring
   - pmg.ssh_hardening
   - pmg.custom_check
   - pmg.security_agents
+  - pmg.cluster_join
   - pmg.auto_updates
   - pmg.audit_logging
   - pmg.pmg_baseline

pmg/node_exporter.sls

@@ -0,0 +1,39 @@
+{% set node_exporter_enabled = salt['pillar.get']('pmg:node_exporter:enabled', True) %}
+{% set node_exporter_package = salt['pillar.get']('pmg:node_exporter:package', 'prometheus-node-exporter') %}
+{% set node_exporter_service = salt['pillar.get']('pmg:node_exporter:service', 'prometheus-node-exporter') %}
+{% set node_exporter_textfile_dir = salt['pillar.get']('pmg:node_exporter:textfile_directory', '/var/lib/prometheus/node-exporter') %}
+{% set node_exporter_args = salt['pillar.get']('pmg:node_exporter:args', '--collector.textfile.directory=' ~ node_exporter_textfile_dir) %}
+
+{% if node_exporter_enabled %}
+pmg-node-exporter-package:
+  pkg.installed:
+    - name: {{ node_exporter_package }}
+
+pmg-node-exporter-textfile-dir:
+  file.directory:
+    - name: {{ node_exporter_textfile_dir }}
+    - user: root
+    - group: root
+    - mode: "0755"
+
+pmg-node-exporter-defaults:
+  file.managed:
+    - name: /etc/default/prometheus-node-exporter
+    - user: root
+    - group: root
+    - mode: "0644"
+    - contents:
+      - "# Managed by Salt."
+      - "ARGS=\"{{ node_exporter_args }}\""
+
+pmg-node-exporter-service:
+  service.running:
+    - name: {{ node_exporter_service }}
+    - enable: true
+    - require:
+      - pkg: pmg-node-exporter-package
+      - file: pmg-node-exporter-textfile-dir
+      - file: pmg-node-exporter-defaults
+    - watch:
+      - file: pmg-node-exporter-defaults
+{% endif %}

pmg/pmg_baseline.sls

@@ -1,4 +1,5 @@
 {% set pmg_services = salt['pillar.get']('pmg:pmg_services', []) %}
+{% set pmg_cluster_only_services = salt['pillar.get']('pmg:pmg_cluster_only_services', ['pmgmirror', 'pmgtunnel']) %}
 {% set manage_hosts_file = salt['pillar.get']('pmg:manage_hosts_file', False) %}
 {% set hosts_file_entries = salt['pillar.get']('pmg:hosts_file_entries', []) %}
 
@@ -7,12 +8,30 @@ pmg-service-{{ svc }}:
   service.running:
     - name: {{ svc }}
     - enable: true
+{% if svc in pmg_cluster_only_services %}
+    - onlyif: "pmgcm status 2>/dev/null | awk 'BEGIN { ok=1 } /no cluster defined/ { ok=0 } END { exit(ok ? 0 : 1) }'"
+{% endif %}
 {% endfor %}
 
 pmg-cluster-status:
   cmd.run:
     - name: pmgcm status
 
+pmg-postfix-identity-from-system-hostname:
+  cmd.run:
+    - name: >-
+        HN="$(hostname -f)" &&
+        DM="${HN#*.}" &&
+        postconf -e "myhostname=${HN}" "mydomain=${DM}" &&
+        systemctl reload postfix
+    - unless: >-
+        HN="$(hostname -f)" &&
+        DM="${HN#*.}" &&
+        [ "$(postconf -h myhostname)" = "${HN}" ] &&
+        [ "$(postconf -h mydomain)" = "${DM}" ]
+    - require:
+      - service: pmg-service-postfix
+
 {% if manage_hosts_file %}
 pmg-managed-hosts:
   file.blockreplace:

pmg/security_agents.sls

@@ -6,6 +6,32 @@
 {% set wazuh_pkg_url = 'https://packages.wazuh.com/4.x/apt/pool/main/w/wazuh-agent/' ~ wazuh_pkg_name %}
 {% set wazuh_pkg_path = '/var/tmp/' ~ wazuh_pkg_name %}
 
+{% set kesl_enabled = salt['pillar.get']('pmg:kesl:enabled', True) %}
+{% set kesl_package_url = salt['pillar.get']('pmg:kesl:package_url', 'https://products.s.kaspersky-labs.com/endpoints/keslinux10/12.4.0.1225/multilanguage-INT-12.4.0.1225/167a2f9c5bdb4e869f4603fbc3a829c1/kesl_12.4.0-1225_amd64.deb') %}
+{% set kesl_package_path = salt['pillar.get']('pmg:kesl:package_path', '/opt/kesl_12.4.0-1225_amd64.deb') %}
+{% set kesl_autoinstall_path = salt['pillar.get']('pmg:kesl:autoinstall_path', '/etc/opt/kaspersky/kesl/autoinstall.ini') %}
+{% set kesl_eula_agreed = salt['pillar.get']('pmg:kesl:eula_agreed', 'Yes') %}
+{% set kesl_privacy_policy_agreed = salt['pillar.get']('pmg:kesl:privacy_policy_agreed', 'Yes') %}
+{% set kesl_use_ksn = salt['pillar.get']('pmg:kesl:use_ksn', 'No') %}
+{% set kesl_group_clean = salt['pillar.get']('pmg:kesl:group_clean', 'No') %}
+{% set kesl_locale = salt['pillar.get']('pmg:kesl:locale', '') %}
+{% set kesl_updater_source = salt['pillar.get']('pmg:kesl:updater_source', 'SCServer') %}
+{% set kesl_install_license = salt['pillar.get']('pmg:kesl:install_license', '') %}
+{% set kesl_update_execute = salt['pillar.get']('pmg:kesl:update_execute', 'No') %}
+{% set kesl_ksvla_mode = salt['pillar.get']('pmg:kesl:ksvla_mode', 'No') %}
+{% set kesl_endpoint_agent_mode = salt['pillar.get']('pmg:kesl:endpoint_agent_mode', 'No') %}
+{% set kesl_setup_bin = '/opt/kaspersky/kesl/bin/kesl-setup.pl' %}
+{% set kesl_setup_marker = '/var/lib/salt/pmg-kesl-autoinstall.done' %}
+
+{% set klnagent_enabled = salt['pillar.get']('pmg:klnagent:enabled', True) %}
+{% set klnagent_package_url = salt['pillar.get']('pmg:klnagent:package_url', 'https://products.s.kaspersky-labs.com/administrationkit/ksc10/16.2.0.1023/russian-INT-29664057-ru/2129c97dba254c9fb8a4161f29a2241c/klnagent64_16.2.0-1023_amd64.deb') %}
+{% set klnagent_package_path = salt['pillar.get']('pmg:klnagent:package_path', '/opt/klnagent64_16.2.0-1023_amd64.deb') %}
+{% set klnagent_server = salt['pillar.get']('pmg:klnagent:administration_server', '10.80.38.76') %}
+{% set klnagent_autoanswers_path = salt['pillar.get']('pmg:klnagent:autoanswers_path', '/etc/opt/kaspersky/klnagent64/autoanswers.conf') %}
+{% set klnagent_autoinstall = salt['pillar.get']('pmg:klnagent:autoinstall', True) %}
+{% set klnagent_eula_accepted = salt['pillar.get']('pmg:klnagent:eula_accepted', True) %}
+{% set klnagent_config_marker = '/var/lib/salt/pmg-klnagent-autoinstall.done' %}
+
 {% set qualys_enabled = salt['pillar.get']('pmg:qualys:enabled', True) %}
 {% set qualys_package_url = salt['pillar.get']('pmg:qualys:package_url', 'https://storage.yandexcloud.net/qualys-agent/QualysCloudAgent.deb') %}
 {% set qualys_package_path = salt['pillar.get']('pmg:qualys:package_path', '/opt/QualysCloudAgent.deb') %}
@@ -13,6 +39,119 @@
 {% set qualys_customer_id = salt['pillar.get']('pmg:qualys:customer_id', '') %}
 {% set qualys_server_uri = salt['pillar.get']('pmg:qualys:server_uri', 'https://qagpublic.qg1.apps.qualys.eu/CloudAgent/') %}
 
+{% if kesl_enabled and kesl_package_path %}
+pmg-kesl-download:
+  cmd.run:
+    - name: wget -qO {{ kesl_package_path }} {{ kesl_package_url }}
+    - unless: test -f {{ kesl_package_path }}
+
+pmg-kesl-autoinstall-config:
+  file.managed:
+    - name: {{ kesl_autoinstall_path }}
+    - user: root
+    - group: root
+    - mode: '0600'
+    - makedirs: true
+    - contents:
+      - "KSVLA_MODE={{ kesl_ksvla_mode }}"
+      - "ENDPOINT_AGENT_MODE={{ kesl_endpoint_agent_mode }}"
+      - "EULA_AGREED={{ kesl_eula_agreed }}"
+      - "PRIVACY_POLICY_AGREED={{ kesl_privacy_policy_agreed }}"
+      - "USE_KSN={{ kesl_use_ksn }}"
+      - "GROUP_CLEAN={{ kesl_group_clean }}"
+{% if kesl_locale %}
+      - "LOCALE={{ kesl_locale }}"
+{% endif %}
+{% if kesl_install_license %}
+      - "INSTALL_LICENSE={{ kesl_install_license }}"
+{% endif %}
+      - "UPDATER_SOURCE={{ kesl_updater_source }}"
+      - "UPDATE_EXECUTE={{ kesl_update_execute }}"
+    - require:
+      - cmd: pmg-kesl-install
+
+pmg-kesl-install:
+  cmd.run:
+    - name: apt-get install -y {{ kesl_package_path }}
+    - onlyif: test -f {{ kesl_package_path }}
+    - unless: dpkg-query -W -f='${Status}' kesl 2>/dev/null | grep -q 'install ok installed'
+    - require:
+      - cmd: pmg-kesl-download
+
+pmg-kesl-setup:
+  cmd.run:
+    - name: mkdir -p /var/lib/salt && {{ kesl_setup_bin }} --autoinstall={{ kesl_autoinstall_path }} && touch {{ kesl_setup_marker }}
+    - onlyif: test -x {{ kesl_setup_bin }}
+    - unless: test -f {{ kesl_setup_marker }}
+    - require:
+      - cmd: pmg-kesl-install
+      - file: pmg-kesl-autoinstall-config
+
+pmg-kesl-service:
+  service.running:
+    - name: kesl
+    - enable: true
+    - require:
+      - cmd: pmg-kesl-install
+      - cmd: pmg-kesl-setup
+{% elif kesl_enabled %}
+pmg-kesl-skipped:
+  test.show_notification:
+    - text: "Kaspersky Endpoint Security skipped: set package_url + package_path in pillar."
+{% endif %}
+
+{% if klnagent_enabled and klnagent_package_path and klnagent_server %}
+pmg-klnagent-download:
+  cmd.run:
+    - name: wget -qO {{ klnagent_package_path }} {{ klnagent_package_url }}
+    - unless: test -f {{ klnagent_package_path }}
+
+pmg-klnagent-autoanswers:
+  file.managed:
+    - name: {{ klnagent_autoanswers_path }}
+    - user: root
+    - group: root
+    - mode: '0600'
+    - makedirs: true
+    - contents:
+      - "KLNAGENT_SERVER={{ klnagent_server }}"
+      - "KLNAGENT_AUTOINSTALL={{ '1' if klnagent_autoinstall else '0' }}"
+      - "EULA_ACCEPTED={{ '1' if klnagent_eula_accepted else '0' }}"
+
+pmg-klnagent-install:
+  cmd.run:
+    - name: DEBIAN_FRONTEND=noninteractive KLAUTOANSWERS={{ klnagent_autoanswers_path }} apt-get install -y {{ klnagent_package_path }}
+    - onlyif: test -f {{ klnagent_package_path }}
+    - unless: dpkg-query -W -f='${Status}' klnagent64 2>/dev/null | grep -q 'install ok installed'
+    - require:
+      - cmd: pmg-klnagent-download
+      - file: pmg-klnagent-autoanswers
+
+pmg-klnagent-autoinstall-configure:
+  cmd.run:
+    - name: >-
+        mkdir -p /var/lib/salt &&
+        DEBIAN_FRONTEND=noninteractive KLAUTOANSWERS={{ klnagent_autoanswers_path }} apt-get install -y --reinstall {{ klnagent_package_path }} &&
+        touch {{ klnagent_config_marker }}
+    - onlyif: dpkg-query -W -f='${Status}' klnagent64 2>/dev/null | grep -q 'install ok installed'
+    - unless: test -f {{ klnagent_config_marker }}
+    - require:
+      - cmd: pmg-klnagent-install
+      - file: pmg-klnagent-autoanswers
+
+pmg-klnagent-service:
+  service.running:
+    - name: klnagent64
+    - enable: true
+    - require:
+      - cmd: pmg-klnagent-install
+      - cmd: pmg-klnagent-autoinstall-configure
+{% elif klnagent_enabled %}
+pmg-klnagent-skipped:
+  test.show_notification:
+    - text: "Kaspersky Administration Agent skipped: set package_url + package_path + administration_server in pillar."
+{% endif %}
+
 {% if wazuh_enabled %}
 pmg-wazuh-download:
   cmd.run:

pmg/smtp_monitoring.sls

@@ -0,0 +1,94 @@
+{% set smtp_monitoring_enabled = salt['pillar.get']('pmg:smtp_monitoring:enabled', True) %}
+{% set blackbox_package = salt['pillar.get']('pmg:smtp_monitoring:blackbox:package', 'prometheus-blackbox-exporter') %}
+{% set blackbox_service = salt['pillar.get']('pmg:smtp_monitoring:blackbox:service', 'prometheus-blackbox-exporter') %}
+{% set smtp_ports = salt['pillar.get']('pmg:smtp_monitoring:ports', [25, 26]) %}
+{% set smtp_log_window_seconds = salt['pillar.get']('pmg:smtp_monitoring:log_window_seconds', 300) %}
+
+{% if smtp_monitoring_enabled %}
+pmg-smtp-monitoring-packages:
+  pkg.installed:
+    - pkgs:
+      - {{ blackbox_package }}
+      - iproute2
+      - jq
+
+pmg-smtp-monitoring-metrics-dir:
+  file.directory:
+    - name: /var/lib/prometheus/node-exporter
+    - user: root
+    - group: root
+    - mode: "0755"
+
+pmg-smtp-monitoring-script:
+  file.managed:
+    - name: /usr/local/bin/pmg-smtp-metrics.sh
+    - source: salt://pmg/files/pmg-smtp-metrics.sh.jinja
+    - template: jinja
+    - context:
+        smtp_ports: {{ smtp_ports }}
+        smtp_log_window_seconds: {{ smtp_log_window_seconds }}
+    - user: root
+    - group: root
+    - mode: "0755"
+    - require:
+      - file: pmg-smtp-monitoring-metrics-dir
+
+pmg-smtp-monitoring-service:
+  file.managed:
+    - name: /etc/systemd/system/pmg-smtp-metrics.service
+    - source: salt://pmg/files/pmg-smtp-metrics.service.jinja
+    - user: root
+    - group: root
+    - mode: "0644"
+
+pmg-smtp-monitoring-timer:
+  file.managed:
+    - name: /etc/systemd/system/pmg-smtp-metrics.timer
+    - source: salt://pmg/files/pmg-smtp-metrics.timer.jinja
+    - user: root
+    - group: root
+    - mode: "0644"
+
+pmg-smtp-monitoring-blackbox-config:
+  file.managed:
+    - name: /etc/prometheus/blackbox.yml
+    - source: salt://pmg/files/blackbox.yml.jinja
+    - template: jinja
+    - context:
+        smtp_ports: {{ smtp_ports }}
+    - user: root
+    - group: root
+    - mode: "0644"
+    - require:
+      - pkg: pmg-smtp-monitoring-packages
+
+pmg-smtp-monitoring-systemd-reload:
+  cmd.run:
+    - name: systemctl daemon-reload
+    - onchanges:
+      - file: pmg-smtp-monitoring-service
+      - file: pmg-smtp-monitoring-timer
+
+pmg-smtp-monitoring-blackbox-service:
+  service.running:
+    - name: {{ blackbox_service }}
+    - enable: true
+    - require:
+      - pkg: pmg-smtp-monitoring-packages
+    - watch:
+      - file: pmg-smtp-monitoring-blackbox-config
+
+pmg-smtp-monitoring-timer-running:
+  service.running:
+    - name: pmg-smtp-metrics.timer
+    - enable: true
+    - require:
+      - cmd: pmg-smtp-monitoring-systemd-reload
+      - file: pmg-smtp-monitoring-script
+
+pmg-smtp-monitoring-run-once:
+  cmd.run:
+    - name: /usr/local/bin/pmg-smtp-metrics.sh
+    - require:
+      - file: pmg-smtp-monitoring-script
+{% endif %}

pmg/verify.sls

@@ -1,9 +1,13 @@
 {% set pmg_services = salt['pillar.get']('pmg:pmg_services', []) %}
+{% set pmg_cluster_only_services = salt['pillar.get']('pmg:pmg_cluster_only_services', ['pmgmirror', 'pmgtunnel']) %}
 
 {% for svc in pmg_services %}
 pmg-verify-service-{{ svc }}:
   cmd.run:
     - name: systemctl is-active --quiet {{ svc }}
+{% if svc in pmg_cluster_only_services %}
+    - onlyif: "pmgcm status 2>/dev/null | awk 'BEGIN { ok=1 } /no cluster defined/ { ok=0 } END { exit(ok ? 0 : 1) }'"
+{% endif %}
 {% endfor %}
 
 pmg-verify-cluster:
@@ -13,3 +17,15 @@ pmg-verify-cluster:
 pmg-verify-sshd-config:
   cmd.run:
     - name: sshd -t
+
+pmg-verify-blackbox-service:
+  cmd.run:
+    - name: systemctl is-active --quiet prometheus-blackbox-exporter
+
+pmg-verify-smtp-metrics-timer:
+  cmd.run:
+    - name: systemctl is-active --quiet pmg-smtp-metrics.timer
+
+pmg-verify-smtp-metrics-exposed:
+  cmd.run:
+    - name: "curl -fsS http://127.0.0.1:9100/metrics | awk 'BEGIN { ok=0 } /^pmg_smtp_sent_recent / { ok=1 } END { exit(ok ? 0 : 1) }'"

scripts/cluster-bootstrap.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 3 ]]; then
+  echo "usage: $0 <target_host> <master_host> <master_ip>" >&2
+  exit 1
+fi
+
+target_host="$1"
+master_host="$2"
+master_ip="$3"
+
+ssh "root@${target_host}" "mkdir -p /root/.ssh && chmod 700 /root/.ssh && [ -f /root/.ssh/id_ed25519 ] || ssh-keygen -t ed25519 -N '' -f /root/.ssh/id_ed25519 >/dev/null"
+
+pubkey_b64="$(ssh "root@${target_host}" "python3 -c 'import base64, pathlib; print(base64.b64encode(pathlib.Path(\"/root/.ssh/id_ed25519.pub\").read_bytes()).decode())'")"
+
+ssh "root@${master_host}" "python3 -c 'import base64, pathlib; pub = base64.b64decode(\"${pubkey_b64}\").decode().strip(); ssh_dir = pathlib.Path(\"/root/.ssh\"); ssh_dir.mkdir(mode=0o700, parents=True, exist_ok=True); auth = ssh_dir / \"authorized_keys\"; lines = auth.read_text().splitlines() if auth.exists() else []; auth.write_text((\"\\n\".join(lines + [pub]) + \"\\n\") if pub not in lines else ((\"\\n\".join(lines) + \"\\n\") if lines else \"\")); auth.chmod(0o600)'"
+
+ssh "root@${target_host}" "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new root@${master_ip} 'exit 0'"