<timestamp>Tuesday, Apr 28, 2026, 3:37 PM (UTC+3)</timestamp>
<user_query>
у меня сейчас такой дашборд, датасорс в таком виде работает
вот это точно не надо (${datasource})
{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "prometheus",
          "uid": "fewcox453ve2od"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Hashicorp Vault Metrics",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": 80625,
  "links": [],
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "area"
            }
          },
          "decimals": 0,
          "mappings": [],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red"
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 56,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "vault_autopilot_healthy",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Overall Cluster Health (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "area"
            }
          },
          "decimals": 0,
          "mappings": [],
          "max": 2,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red"
              },
              {
                "color": "yellow",
                "value": 0
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 24,
        "x": 0,
        "y": 5
      },
      "id": 57,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "vault_autopilot_failure_tolerance",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Failure Tolerance (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 59,
            "gradientMode": "hue",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineStyle": {
              "fill": "solid"
            },
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "area"
            }
          },
          "decimals": 0,
          "mappings": [],
          "max": 5,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red"
              },
              {
                "color": "yellow",
                "value": 3
              },
              {
                "color": "green",
                "value": 4
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 10,
        "w": 24,
        "x": 0,
        "y": 10
      },
      "id": 58,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "vault_autopilot_node_healthy",
          "hide": false,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Cluster Health per Node (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Number of Leases across namespaces",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 0,
        "y": 20
      },
      "id": 67,
      "options": {
        "colorMode": "none",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_expire_num_leases)",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Leases (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Average time to login (ms)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue"
              }
            ]
          },
          "unit": "ms"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 9,
        "x": 3,
        "y": 20
      },
      "id": 68,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(rate(vault_core_handle_login_request_sum[$interval]) / rate(vault_core_handle_login_request_count[$interval]))",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Average Login Time (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 12,
        "y": 20
      },
      "id": 69,
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "max"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "max(idelta(vault_audit_log_request_failure[$interval]))",
          "format": "time_series",
          "interval": "",
          "legendFormat": "Request",
          "refId": "A"
        }
      ],
      "title": "Log Request Failures (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Average core and log request handlers",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "Requests per second",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 9,
        "x": 15,
        "y": 20
      },
      "id": 70,
      "options": {
        "alertThreshold": true,
        "legend": {
          "calcs": [
            "mean",
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "rate(vault_audit_log_request_sum[$__interval])/rate(vault_audit_log_request_count[$__interval])",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "Audit Request",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "rate(vault_audit_log_response_sum[$__interval])/rate(vault_audit_log_response_count[$__interval])",
          "format": "time_series",
          "hide": false,
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "Audit Response",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "rate(vault_core_handle_request_sum[$__interval])/rate(vault_core_handle_request_count[$__interval])",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "Handled by Vault Core",
          "refId": "C"
        }
      ],
      "title": "Requests (req/s) (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Number of KV Engines across namespaces",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 0,
        "y": 24
      },
      "id": 71,
      "maxDataPoints": 100,
      "options": {
        "colorMode": "none",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "count(vault_secret_kv_count)",
          "format": "table",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "KV Engines (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Rate of Authentication Requests",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "fixed"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 8,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 9,
        "x": 3,
        "y": 24
      },
      "id": 72,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(rate(vault_core_handle_login_request_count[$interval]))",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Authentication Rate (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 12,
        "y": 24
      },
      "id": 73,
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "max"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "max(idelta(vault_audit_log_response_failure[$interval]))",
          "format": "time_series",
          "interval": "",
          "legendFormat": "Request",
          "refId": "A"
        }
      ],
      "title": "Log Response Failures (${datasource})",
      "type": "stat"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 28
      },
      "id": 74,
      "panels": [],
      "title": "Identity (${datasource})",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Number of Entities across namespaces",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 12,
        "x": 0,
        "y": 29
      },
      "id": 75,
      "options": {
        "colorMode": "none",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_identity_num_entities)",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Entities (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Number of Entity Aliases across namespaces",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 12,
        "x": 12,
        "y": 29
      },
      "id": 76,
      "options": {
        "colorMode": "none",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_identity_entity_alias_count)",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Entity Aliases (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Entity Count by Namespace",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "left",
            "cellOptions": {
              "type": "auto"
            },
            "filterable": true,
            "inspect": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 32
      },
      "id": 77,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum by (namespace) (vault_identity_entity_count)",
          "format": "table",
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Entity Count by Namespace (${datasource})",
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true
            },
            "indexByName": {},
            "renameByName": {
              "Value": "Entity Count",
              "namespace": "Namespace"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            }
          },
          "mappings": []
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 32
      },
      "id": 78,
      "maxDataPoints": 1,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "pieType": "pie",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "expr": "avg without(instance) (vault_identity_entity_alias_count)",
          "format": "time_series",
          "interval": "",
          "legendFormat": "{{ auth_method }}",
          "refId": "A"
        }
      ],
      "title": "Entity Aliases by Method (${datasource})",
      "type": "piechart"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 39
      },
      "id": 79,
      "panels": [],
      "title": "Token (${datasource})",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "fixed"
          },
          "mappings": [],
          "min": 0,
          "noValue": "N/A",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 0,
        "y": 40
      },
      "id": 80,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_token_count)",
          "format": "time_series",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Available Tokens (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "fixed"
          },
          "mappings": [],
          "min": 0,
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 3,
        "y": 40
      },
      "id": 81,
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "horizontal",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "avg(vault_token_create_count - vault_token_store_count)",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Pending Tokens (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 6,
        "y": 40
      },
      "id": 82,
      "options": {
        "alertThreshold": true,
        "displayMode": "gradient",
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "maxVizHeight": 300,
        "minVizHeight": 16,
        "minVizWidth": 8,
        "namePlacement": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true,
        "sizing": "auto",
        "valueMode": "color"
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "expr": "avg without(instance) (vault_token_count_by_ttl)",
          "format": "time_series",
          "interval": "",
          "legendFormat": "{{ creation_ttl }}",
          "refId": "A"
        }
      ],
      "title": "Tokens by TTL (${datasource})",
      "type": "bargauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Token Count by Namespace",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "left",
            "cellOptions": {
              "type": "auto"
            },
            "filterable": true,
            "inspect": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 16,
        "w": 12,
        "x": 12,
        "y": 40
      },
      "id": 83,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum without(instance) (vault_token_count)",
          "format": "table",
          "instant": true,
          "interval": "",
          "legendFormat": "{{ namespace }}",
          "refId": "A"
        }
      ],
      "title": "Tokens by Namespace (${datasource})",
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Value #A": false,
              "cluster": true,
              "host": true,
              "job": true
            },
            "indexByName": {},
            "renameByName": {
              "Value #A": "Tokens",
              "host": "",
              "namespace": "Namespace"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Token Lookup Rate",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "fixed"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "Lookups / s",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 8,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          },
          "unit": "none"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Lookup"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#0a50a1",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 0,
        "y": 43
      },
      "id": 84,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "rate(vault_token_lookup_count[$interval])",
          "hide": false,
          "interval": "",
          "legendFormat": "{{ host }}",
          "refId": "B"
        }
      ],
      "title": "Token Lookups (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 0,
        "y": 48
      },
      "id": 85,
      "options": {
        "alertThreshold": true,
        "displayMode": "gradient",
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "maxVizHeight": 300,
        "minVizHeight": 16,
        "minVizWidth": 8,
        "namePlacement": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true,
        "sizing": "auto",
        "valueMode": "color"
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "expr": "avg without(instance) (vault_token_count_by_policy)",
          "interval": "",
          "legendFormat": "{{ policy }}",
          "refId": "A"
        }
      ],
      "title": "Tokens by Policy (${datasource})",
      "type": "bargauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 6,
        "x": 6,
        "y": 48
      },
      "id": 86,
      "options": {
        "alertThreshold": true,
        "displayMode": "gradient",
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "maxVizHeight": 300,
        "minVizHeight": 16,
        "minVizWidth": 8,
        "namePlacement": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true,
        "sizing": "auto",
        "valueMode": "color"
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "avg without(instance) (vault_token_count_by_auth)",
          "interval": "",
          "legendFormat": "{{ auth_method }}",
          "refId": "A"
        }
      ],
      "title": "Tokens by Auth Method (${datasource})",
      "type": "bargauge"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 56
      },
      "id": 87,
      "panels": [],
      "title": "Policy (${datasource})",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 0,
        "y": 57
      },
      "id": 88,
      "options": {
        "alertThreshold": true,
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "expr": "avg(irate(vault_policy_set_policy_count[1m]))",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "SET",
          "refId": "A"
        }
      ],
      "title": "Policy Set (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 57
      },
      "id": 89,
      "options": {
        "alertThreshold": true,
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "expr": "avg(irate(vault_policy_get_policy_count[1m]))",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "GET",
          "refId": "A"
        }
      ],
      "title": "Policy Get (${datasource})",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 63
      },
      "id": 90,
      "panels": [],
      "title": "Vault and Raft Leadership (${datasource})",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Any count greater than zero means that Vault experienced a leadership change.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "thresholds"
          },
          "mappings": [],
          "min": 0,
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "#EAB839",
                "value": 1
              },
              {
                "color": "red",
                "value": 2
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 0,
        "y": 64
      },
      "id": 91,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "vault_core_leadership_lost_count",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Vault Leadership Lost Count (${datasource})",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Last reported Vault and Raft leader node",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "super-light-blue",
            "mode": "fixed"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 4,
        "y": 64
      },
      "id": 92,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "name",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "vault_raft_state_leader",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{ host }}",
          "refId": "A"
        }
      ],
      "title": "Current Leader (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Increments whenever raft server starts an election. If there are frequent elections or leadership changes, it would likely indicate network issues between the raft nodes, or that the raft servers themselves are unable to keep up with the load.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "fixed"
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 8,
        "y": 64
      },
      "id": 93,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_raft_state_candidate)",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Vault Raft Leader Elections (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Measures the time since the leader was last able to contact the follower nodes when checking its leader lease",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "fixed"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          },
          "unit": "ms"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 64
      },
      "id": 94,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(rate(vault_raft_leader_lastContact_sum[$interval]) / rate(vault_raft_leader_lastContact_count[$interval]))",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Raft Leader Last Contact (ms) (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Number of Vault cluster leadership setup failures. Failures indicate bad overall cluster leadership status.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "thresholds"
          },
          "mappings": [],
          "min": 0,
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "#EAB839",
                "value": 1
              },
              {
                "color": "red",
                "value": 2
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 0,
        "y": 67
      },
      "id": 95,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "vault_core_leadership_setup_failed",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Vault Leadership Setup Failures (${datasource})",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Number of times node has entered the follower mode. This happens when a new node joins the cluster or after the end of a leader election.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "continuous-BlPu"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 8,
        "x": 4,
        "y": 67
      },
      "id": 96,
      "options": {
        "displayMode": "lcd",
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "maxVizHeight": 300,
        "minVizHeight": 16,
        "minVizWidth": 8,
        "namePlacement": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true,
        "sizing": "auto",
        "text": {},
        "valueMode": "color"
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "vault_raft_state_follower",
          "interval": "",
          "legendFormat": "{{ host }}",
          "refId": "A"
        }
      ],
      "title": "Raft Follower State Entered Count (${datasource})",
      "type": "bargauge"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 70
      },
      "id": 97,
      "panels": [],
      "title": "Raft Integrated Storage Performance (${datasource})",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Performance of Raft Integrated Storage Backend",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 8,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              }
            ]
          },
          "unit": "ms"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 71
      },
      "id": 98,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(rate(vault_raft_storage_delete_sum[$interval]) / rate(vault_raft_storage_delete_count[$interval]))",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "legendFormat": "AVG DEL",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(rate(vault_raft_storage_list_sum[$interval]) / rate(vault_raft_storage_list_count[$interval]))",
          "hide": false,
          "interval": "",
          "legendFormat": "AVG LIST",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(rate(vault_raft_storage_get_sum[$interval]) / rate(vault_raft_storage_get_count[$interval]))",
          "hide": false,
          "interval": "",
          "legendFormat": "AVG GET",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(rate(vault_raft_storage_put_sum[$interval]) / rate(vault_raft_storage_put_count[$interval]))",
          "hide": false,
          "interval": "",
          "legendFormat": "AVG PUT",
          "refId": "D"
        }
      ],
      "title": "Storage Performance (ms) (${datasource})",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "Sum of time spent to perform actions on the storage",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "continuous-BlPu"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue"
              }
            ]
          },
          "unit": "ms"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 71
      },
      "id": 99,
      "options": {
        "displayMode": "lcd",
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "maxVizHeight": 300,
        "minVizHeight": 16,
        "minVizWidth": 8,
        "namePlacement": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true,
        "sizing": "auto",
        "text": {},
        "valueMode": "color"
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_raft_storage_delete_sum)",
          "interval": "",
          "legendFormat": "DEL",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_raft_storage_list_sum)",
          "hide": false,
          "interval": "",
          "legendFormat": "LIST",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_raft_storage_get_sum)",
          "hide": false,
          "interval": "",
          "legendFormat": "GET",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "exemplar": true,
          "expr": "sum(vault_raft_storage_put_sum)",
          "hide": false,
          "interval": "",
          "legendFormat": "PUT",
          "refId": "D"
        }
      ],
      "title": "Time Spent (ms) (${datasource})",
      "type": "bargauge"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 79
      },
      "id": 100,
      "panels": [],
      "title": "CPU/Mem Info (${datasource})",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "noValue": "N/A",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 0.2
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 0,
        "y": 80
      },
      "id": 101,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "expr": "avg(vault_runtime_heap_objects{} / vault_runtime_malloc_count{})",
          "interval": "",
          "intervalFactor": 10,
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Heap Objects Used (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "noValue": "N/A",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "#EAB839",
                "value": 70
              },
              {
                "color": "#EF843C",
                "value": 100
              },
              {
                "color": "#E24D42",
                "value": 150
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 3,
        "y": 80
      },
      "id": 102,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "last"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "text": {},
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "expr": "avg(vault_runtime_num_goroutines{})",
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Number of Goroutines (${datasource})",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "fewcox453ve2od"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green"
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 18,
        "x": 6,
        "y": 80
      },
      "id": 103,
      "options": {
        "alertThreshold": true,
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "11.6.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "fewcox453ve2od"
          },
          "expr": "avg(vault_runtime_alloc_bytes{})",
          "interval": "",
          "intervalFactor": 5,
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Allocated MB (${datasource})",
      "type": "timeseries"
    }
  ],
  "preload": false,
  "refresh": false,
  "schemaVersion": 41,
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "text": "1m",
          "value": "1m"
        },
        "description": "Interval for calculation of averages",
        "hide": 2,
        "label": "Interval",
        "name": "interval",
        "query": "1m",
        "skipUrlSync": true,
        "type": "constant"
      },
      {
        "current": {
          "text": "ITSEC metrics",
          "value": "fewcox453ve2od"
        },
        "includeAll": false,
        "name": "datasource",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "type": "datasource"
      }
    ]
  },
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ]
  },
  "timezone": "",
  "title": "PKI Hashicorp Vault Dashboard",
  "uid": "pki-vault",
  "version": 4
}
</user_query>


Понял: для вашей Grafana не нужен datasource variable вообще. Исправлю наш `monitoring/grafana/vault-pki-dashboard.json` на жёсткий datasource `uid: fewcox453ve2od`, без `${DS_PROMETHEUS}`/`${datasource}`, чтобы импорт работал сразу в текущем контуре.

**Updating dashboard configuration**

I need to respond by updating the dashboard JSON to use the fixed data source UID fewcox453ve2od instead of the current ${datasource} titles. The user mentioned that the current dashboard works with this fixed form, which isn’t ${datasource}. My dashboard uses the ${DS_PROMETHEUS} variable, so I need to edit the monitoring/grafana/vault-pki-dashboard.json to replace the data source UID and potentially remove templating. I should inspect the file titles too.


**Evaluating data source setup**

I need to use a JSON script for replacements, especially since there may be missing data sources in our generated panels. Grafana has this feature where it can inherit panel data sources, and the user sample includes targets. Would it make sense to add that? I’m thinking maybe it’s better to use a Python script to fix the panel and target data sources while removing any templating variables since our file has an empty templating list. I'll need to inspect it further.


Заодно обновлю runbook: в нём сейчас написано «выбрать Prometheus datasource» при импорте, а у нас будет готовый JSON с `uid: fewcox453ve2od`. Если datasource поменяется — нужно будет заменить только этот uid.

**Updating the runbook**

I need to update some documentation. I'll use Python's JSON operation and check if our metrics panels are using the vault_core_handle_request_count metric, which seems to be confirmed. I'll update the runbook to specify choosing a particular UID for the datasource. It's also a good idea to mention that the dashboard fix is related to fewcox. I should think about removing the root_policy_monitoring since it's not relevant anymore. I'll aim for a final concise version.

Shell

python - <<'PY'
import json
from pathlib import Path
p = Path('monitoring/grafana/vault-pki-dashboard.json')
dash = json.loads(p.read_text())
fixed_ds = {'type': 'prometheus', 'uid': 'fewcox453ve2od'}
dash.setdefault('annotations', {})['list'] = [
    {
        'builtIn': 1,
        'datasource': fixed_ds,
        'enable': True,
        'hide': True,
        'iconColor': 'rgba(0, 211, 255, 1)',
        'name': 'Annotations & Alerts',
        'type': 'dashboard',
    }
]
for panel in dash.get('panels', []):
    if panel.get('type') != 'row':
        panel['datasource'] = fixed_ds
    for target in panel.get('targets', []) or []:
        target['datasource'] = fixed_ds
    title = panel.get('title')
    if isinstance(title, str):
        panel['title'] = title.replace(' (${datasource})', '').replace('${datasource}', '').strip()
dash['templating'] = {'list': []}
dash['description'] = 'Vault PKI metrics dashboard'
dash['graphTooltip'] = 1
dash['schemaVersion'] = 41
p.write_text(json.dumps(dash, ensure_ascii=False, indent=2) + '\n')
PY
python -m json.tool "monitoring/grafana/vault-pki-dashboard.json" >/dev/null

Set fixed Grafana datasource UID


ApplyPatch

Input:
*** Begin Patch
*** Update File: /home/ruslanpi/Documents/repos/lmru--devops-core--ansible-vault-oss/docs/vault-pki-monitoring-runbook.md
@@
 1. Grafana → Dashboards → New → Import.
 2. Загрузить `monitoring/grafana/vault-pki-dashboard.json`.
-3. Выбрать Prometheus datasource.
-4. Проверить, что job называется `vault-pki`.
+3. Dashboard уже использует текущий Prometheus datasource Grafana:
+   `uid = fewcox453ve2od`.
+4. Проверить, что job называется `vault-pki`.
+
+Если datasource в Grafana изменится, заменить `fewcox453ve2od` во всём JSON на новый UID datasource. Переменные вида `${datasource}` в dashboard не используются.
*** End Patch


Shell

python -m json.tool "monitoring/grafana/vault-pki-dashboard.json" >/dev/null && yamllint "monitoring/prometheus/vault-pki-alerts.yml"

Validate Grafana JSON and alert YAML


Поправил `monitoring/grafana/vault-pki-dashboard.json` под ваш рабочий формат datasource:

- Везде выставлен fixed datasource:
  ```json
  {
    "type": "prometheus",
    "uid": "fewcox453ve2od"
  }
  ```
- Убрал все переменные вида `${DS_PROMETHEUS}` / `${datasource}`.
- Добавил datasource в `targets`, как в вашем рабочем dashboard.
- Добавил стандартный блок `Annotations & Alerts` с тем же datasource uid.
- В `docs/vault-pki-monitoring-runbook.md` обновил инструкцию: dashboard уже привязан к `uid = fewcox453ve2od`; если datasource поменяется, заменить UID в JSON.

Проверил:

```bash
python -m json.tool monitoring/grafana/vault-pki-dashboard.json
yamllint monitoring/prometheus/vault-pki-alerts.yml
```

Ошибок нет.


<timestamp>Tuesday, Apr 28, 2026, 3:41 PM (UTC+3)</timestamp>
<user_query>
оставил старый, вот так выглядит
datasource
ITSEC metrics

Last 24 hours

Refresh

Overall Cluster Health (ITSEC metrics)
vault_autopilot_healthy{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", owner="itsec", product_id="P1253"}
Failure Tolerance (ITSEC metrics)
vault_autopilot_failure_tolerance{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", owner="itsec", product_id="P1253"}
Cluster Health per Node (ITSEC metrics)
vault_autopilot_node_healthy{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", node_id="p-pki-as-01.lmru.tech", owner="itsec", product_id="P1253"}
vault_autopilot_node_healthy{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", node_id="p-pki-as-02.lmru.tech", owner="itsec", product_id="P1253"}
vault_autopilot_node_healthy{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", node_id="p-pki-as-03.lmru.tech", owner="itsec", product_id="P1253"}
Leases (ITSEC metrics)
1
Average Login Time (ITSEC metrics)
4.50 ms
Log Request Failures (ITSEC metrics)
No data
Requests (req/s) (ITSEC metrics)
Name	Mean	Last *	Max	Min
Handled by Vault Core
4	1	54	0
KV Engines (ITSEC metrics)
No data
Authentication Rate (ITSEC metrics)
Log Response Failures (ITSEC metrics)
No data

Identity (ITSEC metrics)
Entities (ITSEC metrics)
3
Entity Aliases (ITSEC metrics)
3
Entity Count by Namespace (ITSEC metrics)

Namespace

Entity Count

root
3
Entity Aliases by Method (ITSEC metrics)
approle
oidc

Token (ITSEC metrics)
Available Tokens (ITSEC metrics)
1
Pending Tokens (ITSEC metrics)
-1
Tokens by TTL (ITSEC metrics)
1
Tokens by Namespace (ITSEC metrics)

env

hostname

Namespace

owner

product_id

Value

prod
pki.lmru.tech
root
itsec
P1253
1
Token Lookups (ITSEC metrics)
Name
{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", owner="itsec", product_id="P1253"}
Tokens by Policy (ITSEC metrics)
default
1
p0239
1
root_policy_backup
1
Tokens by Auth Method (ITSEC metrics)
approle
1
oidc
1
token
1

Policy (ITSEC metrics)
Policy Set (ITSEC metrics)
SET
Policy Get (ITSEC metrics)
GET

Vault and Raft Leadership (ITSEC metrics)
Vault Leadership Lost Count (ITSEC metrics)
52
Current Leader (ITSEC metrics)
Vault Raft Leader Elections (ITSEC metrics)
894
Raft Leader Last Contact (ms) (ITSEC metrics)
Vault Leadership Setup Failures (ITSEC metrics)
vault_core_leadership_setup_failed{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", owner="itsec", product_id="P1253", quantile="0.5"}
0
vault_core_leadership_setup_failed{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", owner="itsec", product_id="P1253", quantile="0.9"}
0
vault_core_leadership_setup_failed{env="prod", hostname="pki.lmru.tech", instance="pki.lmru.tech:443", job="vault-pki-app", owner="itsec", product_id="P1253", quantile="0.99"}
0
Raft Follower State Entered Count (ITSEC metrics)
721

Raft Integrated Storage Performance (ITSEC metrics)
Storage Performance (ms) (ITSEC metrics)
AVG DEL
AVG LIST
AVG GET
AVG PUT
Time Spent (ms) (ITSEC metrics)
DEL
2.15 mins
LIST
18.0 mins
GET
38.3 mins
PUT
4.58 mins

CPU/Mem Info (ITSEC metrics)
Heap Objects Used (ITSEC metrics)
0.000896
Number of Goroutines (ITSEC metrics)
278
Allocated MB (ITSEC metrics)

avg(vault_runtime_alloc_bytes{})
</user_query>


Ок, тогда считаем старый dashboard основным. По выводу видно, что у вас уже есть рабочий scrape через `job="vault-pki-app"` и `instance="pki.lmru.tech:443"`, поэтому мои rules с `job="vault-pki"` надо привести к вашему фактическому label set, иначе алерты не сработают.

**Responding to user needs**

I need to respond to the user, as they've mentioned keeping the old dashboard and showing the job vault-pki-app. It's unclear if the datasource variable works, so I might need to adapt our documentation to match the old dashboard's context. I should infer that they may need help adjusting alert rules or updating files. Since they indicated “оставил старый” (kept the old), we shouldn't push for changes. I'll need to address limitations in metrics for the old setup as well.
**Updating dashboard and rules**

I don't think we need to use Ask here. I’m considering editing the rules documentation and possibly removing our dashboard. Since the user mentioned they left the old dashboard, it might not be necessary to import ours. Instead, I could update the runbook to advise using the existing dashboard without importing our JSON. I definitely need to inspect current alert rules and think about patching them. Additionally, I'll consider updating the Grafana JSON to ensure compatibility. Progress reporting seems essential here too!


Тогда фиксирую по факту: рабочий job у вас `vault-pki-app`, scrape идёт через `pki.lmru.tech:443`, а здоровье отдельных нод видно не через `up` по каждой ноде, а через `vault_autopilot_node_healthy{node_id=...}`. Поэтому правила и runbook надо переориентировать на `vault_autopilot_*`, а не на мои прямые targets `p-pki-as-*:8200`.

python - <<'PY'
from pathlib import Path
for file in ['monitoring/prometheus/vault-pki-alerts.yml', 'docs/vault-pki-monitoring-runbook.md', 'monitoring/grafana/vault-pki-dashboard.json']:
    p = Path(file)
    data = p.read_text()
    data = data.replace('job="vault-pki"', 'job="vault-pki-app"')
    data = data.replace('job: vault-pki', 'job: vault-pki-app')
    data = data.replace('job называется `vault-pki`', 'job называется `vault-pki-app`')
    p.write_text(data)
PY

Align monitoring job label with existing dashboard


**Troubleshooting replacements**

I'm trying to replace "job=\"vault-pki\"" in a file, but it seems like something went wrong. I realize that in Python, 'job=\"vault-pki\"' actually equals job="vault-pki", so that shouldn’t be the issue. Maybe the working directory is incorrect? Surprisingly, the replacement worked after all. I’m wondering if double quotes are causing confusion. Perhaps I should try using Perl via Shell or even a Python print count to verify things further!

python - <<'PY'
from pathlib import Path
p=Path('monitoring/prometheus/vault-pki-alerts.yml')
data=p.read_text()
print(data.count('job="vault-pki"'))
print(data.count('job="vault-pki-app"'))
data=data.replace('job="vault-pki"','job="vault-pki-app"')
p.write_text(data)
print(p.read_text().count('job="vault-pki"'), p.read_text().count('job="vault-pki-app"'))
PY

Replace alert job labels with actual job


**Checking file freshness**

Hmm, I noticed that the read output seemed old, maybe even stale. I’m double-checking because it said "old," but the count indicates that it’s not actually old. I’m wondering if there might be a caching issue with ReadFile. So, I’ll reread the file to see if I can get the most current information. I need to ensure I’m working with the latest data!


**Investigating shell paths**

I'm noticing some inconsistency with the shell working directory path that lacks a double hyphen. I think the workspace path goes like lmru--devops-core--... The current shell path might be /home/ruslanpi/Documents/repos/lmru--devops-core--ansible-vault-oss. I want to verify if reading the file is producing outdated outputs. It’s a bit perplexing. Maybe using rg through the tool could help clarify this?


**Considering patch availability**

I’m looking at the read file, which feels stale and a bit weird. I think I should continue working on it. There are rules for patch availability that I'm going to need for autopilot. I’m wondering if I should apply the patch using the old string with the previous version. Since we have the current application, I might be able to base the patch on the relevant lines that include that app.

ApplyPatch

Input:
*** Begin Patch
*** Update File: /home/ruslanpi/Documents/repos/lmru--devops-core--ansible-vault-oss/monitoring/prometheus/vault-pki-alerts.yml
@@
-      - alert: VaultNodeDown
+      - alert: VaultEndpointDown
         expr: up{job="vault-pki-app"} == 0
         for: 2m
         labels:
           severity: critical
           service: vault-pki
         annotations:
-          summary: "Vault node is unreachable"
+          summary: "Vault metrics endpoint is unreachable"
           description: "Prometheus cannot scrape {{ $labels.instance }}."

-      - alert: VaultNodeSealed
+      - alert: VaultEndpointSealed
         expr: vault_core_unsealed{job="vault-pki-app"} == 0
         for: 1m
         labels:
           severity: critical
           service: vault-pki
         annotations:
-          summary: "Vault node is sealed"
+          summary: "Vault endpoint reports sealed state"
           description: "{{ $labels.instance }} reports vault_core_unsealed=0."

-      - alert: VaultNoActiveLeader
-        expr: sum(vault_core_active{job="vault-pki-app"}) < 1
+      - alert: VaultAutopilotUnhealthy
+        expr: vault_autopilot_healthy{job="vault-pki-app"} == 0
         for: 2m
         labels:
           severity: critical
           service: vault-pki
         annotations:
-          summary: "Vault cluster has no active leader"
-          description: "No scraped Vault node reports vault_core_active=1."
+          summary: "Vault Autopilot reports unhealthy cluster"
+          description: "Vault Autopilot health is 0 for {{ $labels.instance }}."

-      - alert: VaultTooManyActiveLeaders
-        expr: sum(vault_core_active{job="vault-pki-app"}) > 1
+      - alert: VaultAutopilotNodeUnhealthy
+        expr: vault_autopilot_node_healthy{job="vault-pki-app"} == 0
         for: 2m
         labels:
           severity: critical
           service: vault-pki
         annotations:
-          summary: "Vault cluster reports more than one active leader"
-          description: "More than one Vault node reports vault_core_active=1."
+          summary: "Vault Autopilot reports unhealthy node"
+          description: "Vault node {{ $labels.node_id }} is unhealthy."
+
+      - alert: VaultAutopilotFailureToleranceLow
+        expr: vault_autopilot_failure_tolerance{job="vault-pki-app"} < 1
+        for: 5m
+        labels:
+          severity: warning
+          service: vault-pki
+        annotations:
+          summary: "Vault Autopilot failure tolerance is low"
+          description: "Vault failure tolerance is below 1."
*** End Patch


`ApplyPatch` зацепился за старый контекст, поэтому перепишу alert rules целиком: так надёжнее, и сразу уберу правила, которые не подходят к вашему LB/app scrape.

Shell

python - <<'PY'
from pathlib import Path
Path('monitoring/prometheus/vault-pki-alerts.yml').write_text('''---
groups:
  - name: vault-pki-availability
    rules:
      - alert: VaultEndpointDown
        expr: up{job="vault-pki-app"} == 0
        for: 2m
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault metrics endpoint is unreachable"
          description: "Prometheus cannot scrape {{ $labels.instance }}."

      - alert: VaultEndpointSealed
        expr: vault_core_unsealed{job="vault-pki-app"} == 0
        for: 1m
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault endpoint reports sealed state"
          description: "{{ $labels.instance }} reports vault_core_unsealed=0."

      - alert: VaultAutopilotUnhealthy
        expr: vault_autopilot_healthy{job="vault-pki-app"} == 0
        for: 2m
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault Autopilot reports unhealthy cluster"
          description: "Vault Autopilot health is 0 for {{ $labels.instance }}."

      - alert: VaultAutopilotNodeUnhealthy
        expr: vault_autopilot_node_healthy{job="vault-pki-app"} == 0
        for: 2m
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault Autopilot reports unhealthy node"
          description: "Vault node {{ $labels.node_id }} is unhealthy."

      - alert: VaultAutopilotFailureToleranceLow
        expr: vault_autopilot_failure_tolerance{job="vault-pki-app"} < 1
        for: 5m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault Autopilot failure tolerance is low"
          description: "Vault failure tolerance is below 1."

  - name: vault-pki-raft
    rules:
      - alert: VaultRaftLeaderElectionsHigh
        expr: increase(vault_raft_state_candidate{job="vault-pki-app"}[15m]) > 3
        for: 2m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault Raft leader elections are frequent"
          description: "Vault entered candidate state more than 3 times in 15m."

      - alert: VaultRaftFollowerStateChangesHigh
        expr: increase(vault_raft_state_follower{job="vault-pki-app"}[15m]) > 10
        for: 2m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault Raft follower state changes are frequent"
          description: "Follower state changes increased more than 10 times in 15m."

      - alert: VaultLeadershipLost
        expr: increase(vault_core_leadership_lost_count{job="vault-pki-app"}[15m]) > 0
        for: 1m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault leadership was lost"
          description: "Vault reported leadership lost in the last 15m."

  - name: vault-pki-errors
    rules:
      - alert: VaultApi5xxErrors
        expr: >-
          increase(vault_core_response_status_code{
          job="vault-pki-app",type="5xx"}[5m]) > 0
        for: 1m
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault API returns 5xx errors"
          description: >-
            {{ $labels.instance }} returned 5xx responses in the last 5m.

      - alert: VaultApi4xxSpike
        expr: >-
          increase(vault_core_response_status_code{
          job="vault-pki-app",type="4xx"}[10m]) > 20
        for: 2m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault API 4xx spike"
          description: >-
            {{ $labels.instance }} returned >20 4xx responses in the last 10m.

      - alert: VaultAuditLogRequestFailure
        expr: increase(vault_audit_log_request_failure{job="vault-pki-app"}[5m]) > 0
        for: 1m
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault audit log request failure"
          description: "Vault reports audit log request failures."

      - alert: VaultAuditLogResponseFailure
        expr: increase(vault_audit_log_response_failure{job="vault-pki-app"}[5m]) > 0
        for: 1m
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault audit log response failure"
          description: "Vault reports audit log response failures."

  - name: vault-pki-capacity
    rules:
      - alert: VaultLeaseCountHigh
        expr: vault_expire_num_leases{job="vault-pki-app"} > 10000
        for: 15m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault lease count is high"
          description: "{{ $labels.instance }} has more than 10000 leases."

      - alert: VaultTokenCountHigh
        expr: vault_token_count{job="vault-pki-app"} > 5000
        for: 15m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault token count is high"
          description: "{{ $labels.instance }} has more than 5000 tokens."

      - alert: VaultRuntimeGoroutinesHigh
        expr: vault_runtime_num_goroutines{job="vault-pki-app"} > 5000
        for: 15m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault runtime goroutine count is high"
          description: "{{ $labels.instance }} has more than 5000 goroutines."

  - name: vault-pki-certificates-and-backup
    rules:
      - alert: VaultPkiCaExpiryWarning
        expr: vault_pki_ca_expires_in_seconds < 60 * 24 * 3600
        for: 1h
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault PKI CA expires in less than 60 days"
          description: >-
            Mount {{ $labels.mount }} CA expires in less than 60 days.

      - alert: VaultPkiCaExpiryCritical
        expr: vault_pki_ca_expires_in_seconds < 30 * 24 * 3600
        for: 1h
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault PKI CA expires in less than 30 days"
          description: >-
            Mount {{ $labels.mount }} CA expires in less than 30 days.

      - alert: VaultPkiCaMetricsMissing
        expr: absent(vault_pki_ca_expires_in_seconds{mount="pki-stage"})
        for: 30m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault PKI CA expiry metric is missing"
          description: >-
            Textfile metric vault_pki_ca_expires_in_seconds for pki-stage
            is missing.

      - alert: VaultRaftBackupStale
        expr: time() - vault_pki_backup_latest_mtime_seconds > 2 * 3600
        for: 30m
        labels:
          severity: critical
          service: vault-pki
        annotations:
          summary: "Vault Raft backup is stale"
          description: "Latest local Vault Raft snapshot is older than 2 hours."

      - alert: VaultRaftBackupTooSmall
        expr: vault_pki_backup_latest_size_bytes < 1024 * 1024
        for: 30m
        labels:
          severity: warning
          service: vault-pki
        annotations:
          summary: "Vault Raft backup size is suspiciously small"
          description: "Latest local Vault Raft snapshot is smaller than 1 MiB."
''')
PY

Rewrite alert rules for existing Vault dashboard labels


ApplyPatch

Input:
*** Begin Patch
*** Update File: /home/ruslanpi/Documents/repos/lmru--devops-core--ansible-vault-oss/docs/vault-pki-monitoring-runbook.md
@@
 * Prometheus может читать `/v1/sys/metrics?format=prometheus` **без Vault token**.
 * Доступ к `:8200/v1/sys/metrics` нужно ограничить сетево: только Prometheus/monitoring-сегмент.
-* Собирать метрики нужно с каждого Vault-узла отдельно, не только через LB.
+* В текущем мониторинге уже используется job `vault-pki-app` через `pki.lmru.tech:443`.
+* Здоровье отдельных Vault-нод смотрим через `vault_autopilot_node_healthy{node_id=...}`, а не через `up` по каждой ноде.
@@
-for node in \
-  p-pki-as-01.lmru.tech \
-  p-pki-as-02.lmru.tech \
-  p-pki-as-03.lmru.tech; do
-  echo "== ${node} =="
-  curl -sk "https://${node}:8200/v1/sys/metrics?format=prometheus" \
-    | rg '^(vault_core_active|vault_core_unsealed|vault_raft_storage_follower_last_heartbeat_ms|vault_expire_num_leases|vault_token_count)'
-done
+curl -sk "https://pki.lmru.tech/v1/sys/metrics?format=prometheus" \
+  | rg '^(vault_autopilot_healthy|vault_autopilot_failure_tolerance|vault_autopilot_node_healthy|vault_core_unsealed|vault_expire_num_leases|vault_token_count)'
@@
-* `vault_core_unsealed = 1` на всех узлах;
-* `sum(vault_core_active) = 1` по кластеру;
-* `vault_raft_storage_follower_last_heartbeat_ms` есть на leader-узле для follower-нод;
+* `vault_core_unsealed = 1`;
+* `vault_autopilot_healthy = 1`;
+* `vault_autopilot_failure_tolerance >= 1`;
+* `vault_autopilot_node_healthy = 1` для `p-pki-as-01/02/03`;
 * `vault_core_response_status_code` содержит коды ответов API.
@@
-Добавить job в `prometheus.yml`:
+Текущий рабочий job в Prometheus:

 ```yaml
 scrape_configs:
-  - job_name: vault-pki
+  - job_name: vault-pki-app
     scheme: https
     metrics_path: /v1/sys/metrics
     params:
@@
     scrape_interval: 30s
     scrape_timeout: 10s
     static_configs:
       - targets:
-          - p-pki-as-01.lmru.tech:8200
-          - p-pki-as-02.lmru.tech:8200
-          - p-pki-as-03.lmru.tech:8200
+          - pki.lmru.tech:443
         labels:
-          cluster: pki-lmru
           env: prod
+          hostname: pki.lmru.tech
+          owner: itsec
+          product_id: P1253
@@
-up{job="vault-pki"}
-sum(vault_core_active{job="vault-pki"})
-sum(vault_core_unsealed{job="vault-pki"})
+up{job="vault-pki-app"}
+vault_autopilot_healthy{job="vault-pki-app"}
+vault_autopilot_node_healthy{job="vault-pki-app"}
+vault_autopilot_failure_tolerance{job="vault-pki-app"}
+vault_core_unsealed{job="vault-pki-app"}
@@
-* `VaultNodeDown` — Prometheus не может собрать метрики с узла.
-* `VaultNodeSealed` — Vault sealed.
-* `VaultNoActiveLeader` — нет active leader.
-* `VaultTooManyActiveLeaders` — больше одного leader.
-* `VaultRaftFollowerHeartbeatLag` — follower давно не получает heartbeat.
-* `VaultRaftFollowerIndexLag` — follower отстал по Raft index.
+* `VaultEndpointDown` — Prometheus не может собрать метрики с `pki.lmru.tech:443`.
+* `VaultEndpointSealed` — Vault sealed.
+* `VaultAutopilotUnhealthy` — Autopilot считает кластер нездоровым.
+* `VaultAutopilotNodeUnhealthy` — одна из Vault-нод нездорова (`node_id`).
+* `VaultAutopilotFailureToleranceLow` — отказоустойчивость ниже 1.
+* `VaultRaftLeaderElectionsHigh` — частые выборы leader.
+* `VaultRaftFollowerStateChangesHigh` — частые переходы follower-состояния.
+* `VaultLeadershipLost` — потеря leadership.
@@
-Готовый dashboard:
+Основной dashboard уже есть в Grafana: **PKI Hashicorp Vault Dashboard** (`uid: pki-vault`), datasource **ITSEC metrics** (`uid: fewcox453ve2od`).
+
+Файл ниже оставлен как дополнительный импортируемый вариант, но для текущей эксплуатации можно оставить старый dashboard:
@@
-4. Проверить, что job называется `vault-pki-app`.
+4. Проверить, что job называется `vault-pki-app`.
*** End Patch