From 8eaa7aa47bf1ccb24463ff08c84f062ca064d730 Mon Sep 17 00:00:00 2001 From: Ruslan Piatrovich Date: Wed, 7 Aug 2024 16:57:50 +0300 Subject: [PATCH] first commit --- aqua-notification/aqua-notification-bot.yml | 56 ++++ aqua-notification/aqua-scrape-config.yml | 30 ++ aqua-notification/example-rr.yml | 11 + coroot/PodMonitor.yml | 13 + postgres-exporter.yml | 284 ++++++++++++++++++ victoria-alerts/victoria-alerts-VMAlert.yml | 15 + .../victoria-alerts-VMAlertmanager.yml | 46 +++ victoria-alerts/victoria-alerts-VMRule.yml | 33 ++ victoria-alerts/victoria-alerts-ic.yml | 13 + victoria-alerts/victoria-alerts-main.yml | 150 +++++++++ victoria-alerts/victoria-alerts-ng.yml | 28 ++ 11 files changed, 679 insertions(+) create mode 100644 aqua-notification/aqua-notification-bot.yml create mode 100644 aqua-notification/aqua-scrape-config.yml create mode 100644 aqua-notification/example-rr.yml create mode 100644 coroot/PodMonitor.yml create mode 100644 postgres-exporter.yml create mode 100644 victoria-alerts/victoria-alerts-VMAlert.yml create mode 100644 victoria-alerts/victoria-alerts-VMAlertmanager.yml create mode 100644 victoria-alerts/victoria-alerts-VMRule.yml create mode 100644 victoria-alerts/victoria-alerts-ic.yml create mode 100644 victoria-alerts/victoria-alerts-main.yml create mode 100644 victoria-alerts/victoria-alerts-ng.yml diff --git a/aqua-notification/aqua-notification-bot.yml b/aqua-notification/aqua-notification-bot.yml new file mode 100644 index 0000000..28f1eaa --- /dev/null +++ b/aqua-notification/aqua-notification-bot.yml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: Secret +metadata: + name: aqua-notification-bot-secrets + namespace: aqua + labels: + app: aqua-notification-bot +type: Opaque +data: + API_KEY: bThab2dmeGFNMkFNWjk4Rmh1bWNCanVSUm45RVNjdm4= + MAILBOX: U0NBQHN1cHBvcnQubG1ydS50ZWNo + MAIL_USER: cnUxMDAwXFNDQS1zdmM= + MAIL_PASS: dzhSS0RYaDVaSkdEIXhkRnhldDRaQjJBVUgzOU1nVmpZU256VTNFdENRcllz + SECURITY_BOT_ID: cnhyb29yYWs5dGdldWQzNmg4eXdjeDNnNG8= + SECURITY_BOT_TOKEN: dTQzaXRxb2c5aW5weHBqZnk2azF0Z3B0cmE= + LOOP_CHAT_ID: ZHE4czhranRzYnI5dHB5cTdwcXc4a3U2ZHI= + LOG_LEVEL: REVCVUc= +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: aqua-notification-bot + namespace: aqua + labels: + app: aqua-notification-bot +spec: + replicas: 1 + selector: + matchLabels: + app: aqua-notification-bot + template: + metadata: + labels: + app: aqua-notification-bot + spec: + containers: + - name: aqua-notification-bot + image: docker-local-devops.art.lmru.tech/aqua-notification-bot:latest + ports: + - containerPort: 5000 + envFrom: + - secretRef: + name: aqua-notification-bot-secrets +--- +apiVersion: v1 +kind: Service +metadata: + name: aqua-notification-bot + namespace: aqua +spec: + selector: + app: aqua-notification-bot + ports: + - protocol: TCP + port: 80 + targetPort: 5000 diff --git a/aqua-notification/aqua-scrape-config.yml b/aqua-notification/aqua-scrape-config.yml new file mode 100644 index 0000000..ce13858 --- /dev/null +++ b/aqua-notification/aqua-scrape-config.yml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: aqua-bearer-token + namespace: aqua +type: Opaque +data: + bearer_token: ZWY3MWUwYWJmNTY0YjU3MzYzN2ZkNTFlMmY5MzU5MGY4NGZiZGNkZQ== + +--- +apiVersion: monitoring.coreos.com/v1alpha1 +kind: ScrapeConfig +metadata: + labels: + app: aqua-server-console + prometheus: main + name: aqua-metrics-main + namespace: aqua +spec: + authorization: + credentials: + key: bearer_token + name: aqua-bearer-token + honorLabels: true + metricsPath: /metrics + scheme: HTTP + staticConfigs: + - targets: + - aqua-server-console-svc.aqua.svc.techno.p.mesh.:8080 diff --git a/aqua-notification/example-rr.yml b/aqua-notification/example-rr.yml new file mode 100644 index 0000000..1ad005d --- /dev/null +++ b/aqua-notification/example-rr.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: example-metric +data: + recording-rules.yaml: | + groups: + - name: example_recordings + rules: + - record: example_metric + expr: vector(1) diff --git a/coroot/PodMonitor.yml b/coroot/PodMonitor.yml new file mode 100644 index 0000000..654223c --- /dev/null +++ b/coroot/PodMonitor.yml @@ -0,0 +1,13 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: coroot-node-agent + namespace: coroot + labels: + prometheus: main +spec: + selector: + matchLabels: + app: coroot-node-agent + podMetricsEndpoints: + - port: http diff --git a/postgres-exporter.yml b/postgres-exporter.yml new file mode 100644 index 0000000..da71089 --- /dev/null +++ b/postgres-exporter.yml @@ -0,0 +1,284 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-exporter-secret + namespace: vspspr-prod +type: Opaque +data: + datasource_url_01: cG9zdGdyZXNxbDovL3ByaWNlc19yb191c2VyOmd6NTlIM0Z5RFZuVnU5RlRAcC11cnNhLWEtdnNwc3ByLXByb2QtZGItMDEucnUtY2VudHJhbDEuaW50ZXJuYWw6NTQzNS9wcmljZXM/c3NsbW9kZT1kaXNhYmxl + datasource_url_02: cG9zdGdyZXNxbDovL3ByaWNlc19yb191c2VyOmd6NTlIM0Z5RFZuVnU5RlRAcC11cnNhLWEtdnNwc3ByLXByb2QtZGItMDIucC12c3BzcHIteDEueDEtYS5jbG91ZC5sbXJ1LnRlY2g6NTQzNS9wcmljZXM/c3NsbW9kZT1kaXNhYmxl + datasource_url_03: cG9zdGdyZXNxbDovL3ByaWNlc19yb191c2VyOmd6NTlIM0Z5RFZuVnU5RlRAcC11cnNhLWEtdnNwc3ByLXByb2QtZGItMDMucC12c3BzcHIteDIueDIuY2xvdWQubG1ydS50ZWNoOjU0MzUvcHJpY2VzP3NzbG1vZGU9ZGlzYWJsZQ== + datasource_ip_01: cG9zdGdyZXNxbDovL3ByaWNlc19yb191c2VyOmd6NTlIM0Z5RFZuVnU5RlRAMTAuMjAzLjEzLjIwNzo1NDM1L3ByaWNlcz9zc2xtb2RlPWRpc2FibGU= + datasource_ip_02: cG9zdGdyZXNxbDovL3ByaWNlc19yb191c2VyOmd6NTlIM0Z5RFZuVnU5RlRAMTAuMjAzLjEyNi4xNzA6NTQzNS9wcmljZXM/c3NsbW9kZT1kaXNhYmxl + datasource_ip_03: cG9zdGdyZXNxbDovL3ByaWNlc19yb191c2VyOmd6NTlIM0Z5RFZuVnU5RlRAMTAuMjAzLjExMS4xMzQ6NTQzNS9wcmljZXM/c3NsbW9kZT1kaXNhYmxl +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter-url-01 + namespace: vspspr-prod +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter-url-01 + template: + metadata: + labels: + app: postgres-exporter-url-01 + spec: + containers: + - name: postgres-exporter + image: docker.art.lmru.tech/bitnami/postgres-exporter + args: + - "--web.listen-address=:9187" + env: + - name: DATA_SOURCE_NAME + valueFrom: + secretKeyRef: + name: postgres-exporter-secret + key: datasource_url_01 +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-exporter-url-01 + namespace: vspspr-prod + labels: + prometheus.deckhouse.io/custom-target: postgres-exporter-url + annotations: + prometheus.deckhouse.io/port: "9187" + prometheus.deckhouse.io/allow-unready-pod: "true" +spec: + ports: + - name: http-metrics + port: 9187 + targetPort: 9187 + selector: + app: postgres-exporter-url-01 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter-url-02 + namespace: vspspr-prod +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter-url-02 + template: + metadata: + labels: + app: postgres-exporter-url-02 + spec: + containers: + - name: postgres-exporter + image: docker.art.lmru.tech/bitnami/postgres-exporter + args: + - "--web.listen-address=:9187" + env: + - name: DATA_SOURCE_NAME + valueFrom: + secretKeyRef: + name: postgres-exporter-secret + key: datasource_url_02 +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-exporter-url-02 + namespace: vspspr-prod + labels: + prometheus.deckhouse.io/custom-target: postgres-exporter-url + annotations: + prometheus.deckhouse.io/port: "9187" + prometheus.deckhouse.io/allow-unready-pod: "true" +spec: + ports: + - name: http-metrics + port: 9187 + targetPort: 9187 + selector: + app: postgres-exporter-url-02 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter-url-03 + namespace: vspspr-prod +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter-url-03 + template: + metadata: + labels: + app: postgres-exporter-url-03 + spec: + containers: + - name: postgres-exporter + image: docker.art.lmru.tech/bitnami/postgres-exporter + args: + - "--web.listen-address=:9187" + env: + - name: DATA_SOURCE_NAME + valueFrom: + secretKeyRef: + name: postgres-exporter-secret + key: datasource_url_03 +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-exporter-url-03 + namespace: vspspr-prod + labels: + prometheus.deckhouse.io/custom-target: postgres-exporter-url + annotations: + prometheus.deckhouse.io/port: "9187" + prometheus.deckhouse.io/allow-unready-pod: "true" +spec: + ports: + - name: http-metrics + port: 9187 + targetPort: 9187 + selector: + app: postgres-exporter-url-03 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter-ip-01 + namespace: vspspr-prod +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter-ip-01 + template: + metadata: + labels: + app: postgres-exporter-ip-01 + spec: + containers: + - name: postgres-exporter + image: docker.art.lmru.tech/bitnami/postgres-exporter + args: + - "--web.listen-address=:9187" + env: + - name: DATA_SOURCE_NAME + valueFrom: + secretKeyRef: + name: postgres-exporter-secret + key: datasource_ip_01 +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-exporter-ip-01 + namespace: vspspr-prod + labels: + prometheus.deckhouse.io/custom-target: postgres-exporter-ip + annotations: + prometheus.deckhouse.io/port: "9187" + prometheus.deckhouse.io/allow-unready-pod: "true" +spec: + ports: + - name: http-metrics + port: 9187 + targetPort: 9187 + selector: + app: postgres-exporter-ip-01 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter-ip-02 + namespace: vspspr-prod +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter-ip-02 + template: + metadata: + labels: + app: postgres-exporter-ip-02 + spec: + containers: + - name: postgres-exporter + image: docker.art.lmru.tech/bitnami/postgres-exporter + args: + - "--web.listen-address=:9187" + env: + - name: DATA_SOURCE_NAME + valueFrom: + secretKeyRef: + name: postgres-exporter-secret + key: datasource_ip_02 +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-exporter-ip-02 + namespace: vspspr-prod + labels: + prometheus.deckhouse.io/custom-target: postgres-exporter-ip + annotations: + prometheus.deckhouse.io/port: "9187" + prometheus.deckhouse.io/allow-unready-pod: "true" +spec: + ports: + - name: http-metrics + port: 9187 + targetPort: 9187 + selector: + app: postgres-exporter-ip-02 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter-ip-03 + namespace: vspspr-prod +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter-ip-03 + template: + metadata: + labels: + app: postgres-exporter-ip-03 + spec: + containers: + - name: postgres-exporter + image: docker.art.lmru.tech/bitnami/postgres-exporter + args: + - "--web.listen-address=:9187" + env: + - name: DATA_SOURCE_NAME + valueFrom: + secretKeyRef: + name: postgres-exporter-secret + key: datasource_ip_03 +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-exporter-ip-03 + namespace: vspspr-prod + labels: + prometheus.deckhouse.io/custom-target: postgres-exporter-ip + annotations: + prometheus.deckhouse.io/port: "9187" + prometheus.deckhouse.io/allow-unready-pod: "true" +spec: + ports: + - name: http-metrics + port: 9187 + targetPort: 9187 + selector: + app: postgres-exporter-ip-03 \ No newline at end of file diff --git a/victoria-alerts/victoria-alerts-VMAlert.yml b/victoria-alerts/victoria-alerts-VMAlert.yml new file mode 100644 index 0000000..4bdcf76 --- /dev/null +++ b/victoria-alerts/victoria-alerts-VMAlert.yml @@ -0,0 +1,15 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMAlert +metadata: + name: victoria-alerts-vmalert +spec: + replicaCount: 1 + datasource: + url: "http://vmselect-victoria-alerts-main.devops-core-victoria-alerts.svc.cluster.local:8481/select/0/prometheus" + notifier: + url: "http://vmalertmanager-victoria-alerts-vmalertmanager.devops-core-victoria-alerts.svc.cluster.local:9093" + evaluationInterval: "30s" + ruleSelector: + matchLabels: + role: alert-rules diff --git a/victoria-alerts/victoria-alerts-VMAlertmanager.yml b/victoria-alerts/victoria-alerts-VMAlertmanager.yml new file mode 100644 index 0000000..2b1bb7e --- /dev/null +++ b/victoria-alerts/victoria-alerts-VMAlertmanager.yml @@ -0,0 +1,46 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMAlertmanager +metadata: + name: victoria-alerts-vmalertmanager +spec: + replicaCount: 2 + image: + repository: docker-remote-docker-io.art.lmru.tech/prom/alertmanager + tag: v0.25.0 + pullPolicy: IfNotPresent + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - "alertmananger" + topologyKey: "kubernetes.io/hostname" + tolerations: + - effect: NoExecute + key: dedicated + value: victoria-alerts + nodeSelector: + node-role/victoria-alerts: "" + resources: + requests: + memory: "64Mi" + cpu: "250m" + limits: + memory: "128Mi" + cpu: "500m" + configRawYaml: | + global: + resolve_timeout: 5m + route: + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'webhook' + receivers: + - name: 'webhook' + webhook_configs: + - url: 'http://localhost:30502/' diff --git a/victoria-alerts/victoria-alerts-VMRule.yml b/victoria-alerts/victoria-alerts-VMRule.yml new file mode 100644 index 0000000..96da322 --- /dev/null +++ b/victoria-alerts/victoria-alerts-VMRule.yml @@ -0,0 +1,33 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: victoria-alerts-vmrule-correct-rule +spec: + groups: + - name: vmalert + rules: + - alert: vmalert config reload error + expr: delta(vmalert_config_last_reload_errors_total[5m]) > 0 + for: 10s + labels: + severity: major + job: "{{ $labels.job }}" + annotations: + value: "{{ $value }}" + description: 'error reloading vmalert config, reload count for 5 min {{ $value }}' +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: victoria-alerts-vmrule-incorrect-rule +spec: + groups: + - name: incorrect rule + rules: + - alert: vmalert bad config + expr: bad expression + for: 10s + labels: + severity: major + annotations: + value: "{{ $badValue | bad function }}" \ No newline at end of file diff --git a/victoria-alerts/victoria-alerts-ic.yml b/victoria-alerts/victoria-alerts-ic.yml new file mode 100644 index 0000000..77484a4 --- /dev/null +++ b/victoria-alerts/victoria-alerts-ic.yml @@ -0,0 +1,13 @@ +--- +apiVersion: deckhouse.io/v1 +kind: YandexInstanceClass +metadata: + name: victoria-alerts +spec: + cores: 8 + memory: 16384 + diskSizeGB: 50 + diskType: network-ssd + imageID: fd8emvfmfoaordspe1jr + networkType: Standard + platformID: standard-v2 \ No newline at end of file diff --git a/victoria-alerts/victoria-alerts-main.yml b/victoria-alerts/victoria-alerts-main.yml new file mode 100644 index 0000000..072164e --- /dev/null +++ b/victoria-alerts/victoria-alerts-main.yml @@ -0,0 +1,150 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMCluster +metadata: + name: victoria-alerts-main +spec: + retentionPeriod: "3d" + replicationFactor: 2 + vmstorage: + replicaCount: 3 + logFormat: json + logLevel: "INFO" + image: + repository: docker-remote-docker-io.art.lmru.tech/victoriametrics/vmstorage + tag: v1.97.0-cluster + pullPolicy: IfNotPresent + storageDataPath: "/vm-data" + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: DoNotSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - "vmstorage" + topologyKey: "kubernetes.io/hostname" + tolerations: + - effect: NoExecute + key: dedicated + value: victoria-alerts + nodeSelector: + node-role/victoria-alerts: "" + podDisruptionBudget: + minAvailable: 2 + storage: + volumeClaimTemplate: + spec: + storageClassName: network-ssd + resources: + requests: + storage: 30Gi + resources: + limits: + memory: 4Gi + requests: + memory: 512Mi + cpu: 1 + vmselect: + replicaCount: 3 + serviceSpec: + metadata: + name: victoria-alerts-cluster-main-access + annotations: + yandex.cpi.flant.com/listener-subnet-id: e9b62pbnf0oaj4btj7rc + yandex.cpi.flant.com/listener-address-ipv4: 10.203.199.3 + spec: + type: LoadBalancer + logFormat: json + logLevel: "INFO" + image: + repository: docker-remote-docker-io.art.lmru.tech/victoriametrics/vmselect + tag: v1.97.0-cluster + pullPolicy: IfNotPresent + cacheMountPath: "/select-cache" + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: DoNotSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - "vmselect" + topologyKey: "kubernetes.io/hostname" + tolerations: + - effect: NoExecute + key: dedicated + value: victoria-alerts + nodeSelector: + node-role/victoria-alerts: "" + podDisruptionBudget: + minAvailable: 2 + storage: + volumeClaimTemplate: + spec: + storageClassName: network-ssd + resources: + requests: + storage: 2Gi + resources: + limits: + memory: "1024Mi" + requests: + memory: 256Mi + cpu: 200m + + #https://docs.victoriametrics.com/operator/api/#vminsert + vminsert: + replicaCount: 3 + serviceSpec: + metadata: + name: victoria-alerts-cluster-main-insert + annotations: + yandex.cpi.flant.com/listener-subnet-id: e2l6nn0el6dm1e5p73ki + yandex.cpi.flant.com/listener-address-ipv4: 10.203.19.200 + spec: + type: LoadBalancer + image: + repository: docker-remote-docker-io.art.lmru.tech/victoriametrics/vminsert + tag: v1.97.0-cluster + pullPolicy: IfNotPresent + logFormat: json + logLevel: "INFO" + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: DoNotSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - "vminsert" + topologyKey: "kubernetes.io/hostname" + tolerations: + - effect: NoExecute + key: dedicated + value: victoria-alerts + nodeSelector: + node-role/victoria-alerts: "" + podDisruptionBudget: + minAvailable: 2 + resources: + limits: + memory: "1Gi" + requests: + memory: "256Mi" + cpu: 1 \ No newline at end of file diff --git a/victoria-alerts/victoria-alerts-ng.yml b/victoria-alerts/victoria-alerts-ng.yml new file mode 100644 index 0000000..8c34d40 --- /dev/null +++ b/victoria-alerts/victoria-alerts-ng.yml @@ -0,0 +1,28 @@ +--- +apiVersion: deckhouse.io/v1 +kind: NodeGroup +metadata: + name: victoria-alerts +spec: + nodeType: CloudEphemeral + cloudInstances: + minPerZone: 1 + maxPerZone: 3 + zones: + - ru-central1-a + - ru-central1-b + - ru-central1-d + classReference: + kind: YandexInstanceClass + name: victoria-alerts + nodeTemplate: + labels: + node-role/victoria-alerts: "" + taints: + - effect: NoExecute + key: dedicated + value: victoria-alerts + disruptions: + approvalMode: Automatic + chaos: + mode: Disabled \ No newline at end of file