From f218f3e4ab660786fea3421b2b30dc018a05042a Mon Sep 17 00:00:00 2001 From: Johann Hoffmann Date: Thu, 14 Jul 2022 17:45:20 +0200 Subject: [PATCH] [WIFI-10094] Prometheus alerts are not getting sent to Slack (#213) * Renew webhook URL and remove obsolete resources Signed-off-by: Johann Hoffmann * Remove obsolete secret file reference Signed-off-by: Johann Hoffmann * Re-add Elasticsearch datasource for future use Signed-off-by: Johann Hoffmann --- helmfile/cloud-sdk/helmfile.yaml | 15 +-------------- helmfile/cloud-sdk/secrets/alertmanager.yaml | 10 +++++----- helmfile/cloud-sdk/secrets/grafana.yaml | 17 ----------------- 3 files changed, 6 insertions(+), 36 deletions(-) delete mode 100644 helmfile/cloud-sdk/secrets/grafana.yaml diff --git a/helmfile/cloud-sdk/helmfile.yaml b/helmfile/cloud-sdk/helmfile.yaml index 07dfd20..594eebc 100644 --- a/helmfile/cloud-sdk/helmfile.yaml +++ b/helmfile/cloud-sdk/helmfile.yaml @@ -52,7 +52,6 @@ environments: secrets: - secrets/influxdb.yaml - secrets/sso.yaml - - secrets/grafana.yaml - secrets/alertmanager.yaml - secrets/actions-runner-controller.yaml - secrets/ucentral-ap-firmware-logstash.yaml @@ -248,6 +247,7 @@ releases: area: testbed namespace: "{{`{{ $labels.namespace }}`}}" pod: "{{`{{ $labels.pod }}`}}" + reason: "{{`{{ $labels.reason }}`}}" annotations: title: Pod *{{`{{ $labels.namespace }}/{{ $labels.pod }}`}}* has terminated, the last reason is *{{`{{ $labels.reason }}`}}* description: The pod *{{`{{ $labels.namespace }}/{{ $labels.pod }}`}}* terminated because of the above-mentioned reason, please investigate the cause. @@ -371,19 +371,6 @@ releases: esVersion: 6 timeField: '@timestamp' - notifiers: - notifiers.yaml: - notifiers: - - name: Slack - type: slack - uid: slack - org_id: 1 - is_default: true - frequency: 1h - settings: - url: {{ .Environment.Values.grafana.slack_api_url }} - disable_resolve_message: false - - alertmanager: config: global: diff --git a/helmfile/cloud-sdk/secrets/alertmanager.yaml b/helmfile/cloud-sdk/secrets/alertmanager.yaml index 4b286e0..497532a 100644 --- a/helmfile/cloud-sdk/secrets/alertmanager.yaml +++ b/helmfile/cloud-sdk/secrets/alertmanager.yaml @@ -1,17 +1,17 @@ alertmanager: - slack_api_url: ENC[AES256_GCM,data:1TZv/NfBaSLSKAKf8CbAlPnqoMA4YkQ0N3OCmBMmQ3q+AF5j3bn/zG4VyrPqWe1EZW4A+PbMoWMsDylO4DayA+YIm0DIBjBEyllIJE5X8eCd,iv:0jMMRTlITDK3t0uLNXF7VPvjLP5IICc7CgcUax6akog=,tag:V0aT2FYKzQZ5+KF69vWTbw==,type:str] + slack_api_url: ENC[AES256_GCM,data:roLDZXQ4QF8I6ggZ5Rf43bqij5eq19hya0P5vCeATEL9rNMR32ILG0c2HfTGpGix0QG8q4I0wTg0SDv2nrCQXLgeHWiN8e+rDfd41hoWB+NH,iv:kAI9XTe3uRy3xnH9t3cJnOU7KVn74nMZREATCJnRXrI=,tag:Ryz0aAt90nzyveSRCnCpsQ==,type:str] sops: kms: - arn: arn:aws:kms:us-east-2:289708231103:alias/helm-secrets - created_at: "2021-06-02T09:54:40Z" - enc: AQICAHiG/4CitJjM31GdYxTw9OLz/Zs5oK+DCq0cU2fAjtAA3AEVaTGNQ83beH6ZX3pZJRosAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMIqU54uF1ndMQfZ2yAgEQgDsa2jwmkjW6a/hAZ5uQhz39U243I4Z4RD24jZVxq28zxmdGP1WYfjDuqERCoxbIWiNcVww7CnbBqw2BWg== + created_at: "2022-07-12T15:25:48Z" + enc: AQICAHiG/4CitJjM31GdYxTw9OLz/Zs5oK+DCq0cU2fAjtAA3AF0CEgHHqDXuFq5pR+jXtIMAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMr8FJk/tRAuiHm3G7AgEQgDujiOa9kle9BC8NVsDxpPQt5ev+bDVFx2N6il9OLFXoI8F1zOiqUVyz2k9JtitAKgozAagQRkXetyRPrA== aws_profile: "" gcp_kms: [] azure_kv: [] hc_vault: [] age: [] - lastmodified: "2021-06-02T09:54:41Z" - mac: ENC[AES256_GCM,data:M5yMryXnHApML9bmfSFCPHZZ9JxLfuspw2YQEWBAfclLMXlUlkkRnjePr3TEvgieUpeUAkBE8vmz/nATIjgL3/Cxh55CbtKmXrGYnonVI2O313ATDNvY68gJ5aQZ1d70w39/JwfkM92/XQSK3Gdkoxx9Ph8zQG2BMiFfLgM2m7o=,iv:ef6YMbCV9vQw6zyXmwXGeFPku2zXt0N5FN18kaWVF1w=,tag:/34viwyV2RxeUp7wLw8vcg==,type:str] + lastmodified: "2022-07-12T15:25:49Z" + mac: ENC[AES256_GCM,data:NDKRaAbCehoYRTBvy5k3eNeb5pGsgitiUZd8idKusn8dQyzwy5pd1jxL8AK1Thce0wLf5JbI02GvfioBjfw+El9FfhviEXZGtz5PHKFWMa6trpSPSMlHIeJvAhzL+IWNi/P6ImUjzxaz74hK4UJ9F0Krk8u3LZH3Tt1oSkFQTBw=,iv:2KDMdIQEZTPlEYmChurP44h23GazUtJJ8BMBF22T3ek=,tag:ckV4pWPBa4L24AG4Cx8Vzw==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.1 diff --git a/helmfile/cloud-sdk/secrets/grafana.yaml b/helmfile/cloud-sdk/secrets/grafana.yaml deleted file mode 100644 index 9b8acd2..0000000 --- a/helmfile/cloud-sdk/secrets/grafana.yaml +++ /dev/null @@ -1,17 +0,0 @@ -grafana: - slack_api_url: ENC[AES256_GCM,data:zAswmtSiM9HenATnqKKenxlTxSatSOUL9q5As5zpTgEcmlNnZf8yVRqCBw0Y3QVyOm1EmkO0bkqTclcuOPC7rXnqzLxAhBkM6wR5v0qDbt3W,iv:BOFaXDOi3bivQBTx1g66ZvdXlURSMeYC72RAEYZwpkc=,tag:Z38Y/a1fsSVDyQD3sfth1w==,type:str] -sops: - kms: - - arn: arn:aws:kms:us-east-2:289708231103:alias/helm-secrets - created_at: "2022-07-11T12:54:50Z" - enc: AQICAHiG/4CitJjM31GdYxTw9OLz/Zs5oK+DCq0cU2fAjtAA3AGgr/yv8vhzDeiU/2AFGjmCAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMwcY1HiP4LXObPBjqAgEQgDuP3Ar1XM3tsXDjZnJ+VwwC3aufnq6lw1FYUxwObrJfWXMziok8tPwsUa3cEPoE6Z2rKBO89rA5lT0h9w== - aws_profile: "" - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2022-07-11T12:54:51Z" - mac: ENC[AES256_GCM,data:cvFfZjqfHWcgDq9ocEcjrqM0gdv34Cn+RXYI5iA2o7DlRnD2GWNqb4d78cf5uJeQirAyiHqtJiqGsphUNf78AQf9vofChHutBSshgm5F3CctNEJeXsQQBgRMlAI06EclzYWz8wDWa+HWHOvCX3MsM7jMJDKu4H+/UMzxibl2XqQ=,iv:bdRdjmGyPH1x9UKRYAPHuuh2SaZkfuIRfhbgtcYwpfA=,tag:H3JzFqkdWubvelcbvOj+Eg==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.1