[WIFI-10094] Prometheus alerts are not getting sent to Slack (#213)

* Renew webhook URL and remove obsolete resources

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Remove obsolete secret file reference

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Re-add Elasticsearch datasource for future use

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

helmfile/cloud-sdk/helmfile.yaml

@@ -52,7 +52,6 @@ environments:
     secrets:
       - secrets/influxdb.yaml
       - secrets/sso.yaml
-      - secrets/grafana.yaml
       - secrets/alertmanager.yaml
       - secrets/actions-runner-controller.yaml
       - secrets/ucentral-ap-firmware-logstash.yaml
@@ -248,6 +247,7 @@ releases:
               area: testbed
               namespace: "{{`{{ $labels.namespace }}`}}"
               pod: "{{`{{ $labels.pod }}`}}"
+              reason: "{{`{{ $labels.reason }}`}}"
             annotations:
               title: Pod *{{`{{ $labels.namespace }}/{{ $labels.pod }}`}}* has terminated, the last reason is *{{`{{ $labels.reason }}`}}*
               description: The pod *{{`{{ $labels.namespace }}/{{ $labels.pod }}`}}* terminated because of the above-mentioned reason, please investigate the cause.
@@ -371,19 +371,6 @@ releases:
             esVersion: 6
             timeField: '@timestamp'
 
-      notifiers:
-        notifiers.yaml:
-          notifiers:
-          - name: Slack
-            type: slack
-            uid: slack
-            org_id: 1
-            is_default: true
-            frequency: 1h
-            settings:
-              url: {{ .Environment.Values.grafana.slack_api_url }}
-            disable_resolve_message: false
-
   - alertmanager:
       config:
         global:

helmfile/cloud-sdk/secrets/alertmanager.yaml

@@ -1,17 +1,17 @@
 alertmanager:
-    slack_api_url: ENC[AES256_GCM,data:1TZv/NfBaSLSKAKf8CbAlPnqoMA4YkQ0N3OCmBMmQ3q+AF5j3bn/zG4VyrPqWe1EZW4A+PbMoWMsDylO4DayA+YIm0DIBjBEyllIJE5X8eCd,iv:0jMMRTlITDK3t0uLNXF7VPvjLP5IICc7CgcUax6akog=,tag:V0aT2FYKzQZ5+KF69vWTbw==,type:str]
+    slack_api_url: ENC[AES256_GCM,data:roLDZXQ4QF8I6ggZ5Rf43bqij5eq19hya0P5vCeATEL9rNMR32ILG0c2HfTGpGix0QG8q4I0wTg0SDv2nrCQXLgeHWiN8e+rDfd41hoWB+NH,iv:kAI9XTe3uRy3xnH9t3cJnOU7KVn74nMZREATCJnRXrI=,tag:Ryz0aAt90nzyveSRCnCpsQ==,type:str]
 sops:
     kms:
         - arn: arn:aws:kms:us-east-2:289708231103:alias/helm-secrets
-          created_at: "2021-06-02T09:54:40Z"
-          enc: AQICAHiG/4CitJjM31GdYxTw9OLz/Zs5oK+DCq0cU2fAjtAA3AEVaTGNQ83beH6ZX3pZJRosAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMIqU54uF1ndMQfZ2yAgEQgDsa2jwmkjW6a/hAZ5uQhz39U243I4Z4RD24jZVxq28zxmdGP1WYfjDuqERCoxbIWiNcVww7CnbBqw2BWg==
+          created_at: "2022-07-12T15:25:48Z"
+          enc: AQICAHiG/4CitJjM31GdYxTw9OLz/Zs5oK+DCq0cU2fAjtAA3AF0CEgHHqDXuFq5pR+jXtIMAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMr8FJk/tRAuiHm3G7AgEQgDujiOa9kle9BC8NVsDxpPQt5ev+bDVFx2N6il9OLFXoI8F1zOiqUVyz2k9JtitAKgozAagQRkXetyRPrA==
           aws_profile: ""
     gcp_kms: []
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2021-06-02T09:54:41Z"
-    mac: ENC[AES256_GCM,data:M5yMryXnHApML9bmfSFCPHZZ9JxLfuspw2YQEWBAfclLMXlUlkkRnjePr3TEvgieUpeUAkBE8vmz/nATIjgL3/Cxh55CbtKmXrGYnonVI2O313ATDNvY68gJ5aQZ1d70w39/JwfkM92/XQSK3Gdkoxx9Ph8zQG2BMiFfLgM2m7o=,iv:ef6YMbCV9vQw6zyXmwXGeFPku2zXt0N5FN18kaWVF1w=,tag:/34viwyV2RxeUp7wLw8vcg==,type:str]
+    lastmodified: "2022-07-12T15:25:49Z"
+    mac: ENC[AES256_GCM,data:NDKRaAbCehoYRTBvy5k3eNeb5pGsgitiUZd8idKusn8dQyzwy5pd1jxL8AK1Thce0wLf5JbI02GvfioBjfw+El9FfhviEXZGtz5PHKFWMa6trpSPSMlHIeJvAhzL+IWNi/P6ImUjzxaz74hK4UJ9F0Krk8u3LZH3Tt1oSkFQTBw=,iv:2KDMdIQEZTPlEYmChurP44h23GazUtJJ8BMBF22T3ek=,tag:ckV4pWPBa4L24AG4Cx8Vzw==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.7.1

helmfile/cloud-sdk/secrets/grafana.yaml

@@ -1,17 +0,0 @@
-grafana:
-    slack_api_url: ENC[AES256_GCM,data:zAswmtSiM9HenATnqKKenxlTxSatSOUL9q5As5zpTgEcmlNnZf8yVRqCBw0Y3QVyOm1EmkO0bkqTclcuOPC7rXnqzLxAhBkM6wR5v0qDbt3W,iv:BOFaXDOi3bivQBTx1g66ZvdXlURSMeYC72RAEYZwpkc=,tag:Z38Y/a1fsSVDyQD3sfth1w==,type:str]
-sops:
-    kms:
-        - arn: arn:aws:kms:us-east-2:289708231103:alias/helm-secrets
-          created_at: "2022-07-11T12:54:50Z"
-          enc: AQICAHiG/4CitJjM31GdYxTw9OLz/Zs5oK+DCq0cU2fAjtAA3AGgr/yv8vhzDeiU/2AFGjmCAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMwcY1HiP4LXObPBjqAgEQgDuP3Ar1XM3tsXDjZnJ+VwwC3aufnq6lw1FYUxwObrJfWXMziok8tPwsUa3cEPoE6Z2rKBO89rA5lT0h9w==
-          aws_profile: ""
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age: []
-    lastmodified: "2022-07-11T12:54:51Z"
-    mac: ENC[AES256_GCM,data:cvFfZjqfHWcgDq9ocEcjrqM0gdv34Cn+RXYI5iA2o7DlRnD2GWNqb4d78cf5uJeQirAyiHqtJiqGsphUNf78AQf9vofChHutBSshgm5F3CctNEJeXsQQBgRMlAI06EclzYWz8wDWa+HWHOvCX3MsM7jMJDKu4H+/UMzxibl2XqQ=,iv:bdRdjmGyPH1x9UKRYAPHuuh2SaZkfuIRfhbgtcYwpfA=,tag:H3JzFqkdWubvelcbvOj+Eg==,type:str]
-    pgp: []
-    unencrypted_suffix: _unencrypted
-    version: 3.7.1