[WIFI-10088] Research and install solution to keep coredumps for debugging purposes (#215)

* Add IAM user and bucket for core-dump-handler

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Fix Terraform format

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Add core-dumps-s3 to Atlantis

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Remove outputs.tf and switch to separate S3 ACL resource

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Fix Terraform state key name

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Add core-dump-handler to helmfile

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Commit helmfile.lock

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

* Enable helmfile deployment of core-dump-handler

Signed-off-by: Johann Hoffmann <johann.hoffmann@mailbox.org>

atlantis.yaml

@@ -16,3 +16,5 @@ projects:
   dir: terraform/wifi-289708231103/atlantis
 - name: quali
   dir: terraform/wifi-289708231103/quali
+- name: core-dumps-s3
+  dir: terraform/wifi-289708231103/core-dumps-s3

helmfile/cloud-sdk/helmfile.lock

@@ -15,6 +15,9 @@ dependencies:
 - name: cluster-autoscaler
   repository: https://kubernetes.github.io/autoscaler/
   version: 9.11.0
+- name: core-dump-handler
+  repository: https://ibm.github.io/core-dump-handler
+  version: v8.6.0
 - name: elasticsearch
   repository: https://charts.helm.sh/stable
   version: 1.32.5
@@ -51,5 +54,5 @@ dependencies:
 - name: tigera-operator
   repository: https://projectcalico.docs.tigera.io/charts
   version: v3.22.2
-digest: sha256:678736d718edbcb6c3b145ff75f6f157e4412f790b7e420933a3289ec9cedbe0
+digest: sha256:19f237181943ecc55a944a648240bdc21e78d2795b7d1b1c5cbd4f26b34e6e3a
-generated: "2022-07-18T14:47:51.092045365+03:00"
+generated: "2022-07-21T15:43:59.17512049+02:00"

helmfile/cloud-sdk/helmfile.yaml

@@ -27,6 +27,8 @@ repositories:
   url: https://projectcalico.docs.tigera.io/charts
 - name: braedon
   url: https://braedon.github.io/helm
+- name: core-dump-handler
+  url: https://ibm.github.io/core-dump-handler
 
 environments:
   azure:
@@ -55,6 +57,7 @@ environments:
       - secrets/alertmanager.yaml
       - secrets/actions-runner-controller.yaml
       - secrets/ucentral-ap-firmware-logstash.yaml
+      - secrets/core-dump-handler.yaml
     values:
     - eks:
         clusterName: tip-wlan-main
@@ -98,6 +101,8 @@ environments:
         enabled: true
     - calico:
         enabled: true
+    - core-dump-handler:
+        enabled: true
 
 helmDefaults:
   force: false
@@ -1048,3 +1053,16 @@ releases:
               }
             }
 
+- name: core-dump-handler
+  condition: core-dump-handler.enabled
+  chart: core-dump-handler/core-dump-handler
+  version: v8.6.0
+  namespace: ibm-observe
+  labels:
+    app: core-dump-handler
+  values:
+    - daemonset:
+        s3BucketName: openwifi-core-dumps
+        s3AccessKey: {{ .Environment.Values.core_dump_handler.s3_access_key }}
+        s3Secret: {{ .Environment.Values.core_dump_handler.s3_secret }}
+        s3Region: us-east-1

helmfile/cloud-sdk/secrets/core-dump-handler.yaml

@@ -0,0 +1,18 @@
+core_dump_handler:
+    s3_access_key: ENC[AES256_GCM,data:DXYGNj7zcjwNG4ApJb/XVB9Z5YY=,iv:uWMI9BUCBc+ohJwk6nTDRQK/Yw9misZk3xwGoN7qGpI=,tag:qqnH0RmE9dnnqrdDCTYxdQ==,type:str]
+    s3_secret: ENC[AES256_GCM,data:z77t0hx9hveX1HcurZEP0IExm5GRQAi3BYC6yq6fQljBg3UN2+RSQA==,iv:DgxjUQYWoXm0saEkUHg/vDMX+XmHSff7pewUnoEKHbY=,tag:zP2yJKihBCkeR41t6d4cxw==,type:str]
+sops:
+    kms:
+        - arn: arn:aws:kms:us-east-2:289708231103:alias/helm-secrets
+          created_at: "2022-07-21T13:08:08Z"
+          enc: AQICAHiG/4CitJjM31GdYxTw9OLz/Zs5oK+DCq0cU2fAjtAA3AFPg3cG7BFFLgkGt+Ti/zEgAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMH347Crtx32/LE/q6AgEQgDsJVGzZWfuYP5HQbC0CD0XErUY8QFw21/wUUxBOKM+f+aZr+ZeKyL9EyyivhIkW/rEvzLEQzPiGxBOZEQ==
+          aws_profile: ""
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2022-07-21T13:08:10Z"
+    mac: ENC[AES256_GCM,data:IW295zTIfSxKxb/M0Oy1EjpcAZNljzo9QzOhZrqgyxsCCBY5L2pplimQrtiUz4EHTy/MAwMu6bVLax0Cbt/H8anIhAHeCGUh+qLB0dMYHOVwcRRCQVVrFYOl/cN3znjP7UPkbHCJITUNw9Q/1WfgcRNfrrBgRU6szdsBFMKMkEU=,iv:nEUDmm0HmvN+3Rbbj3ngIe7pJ2sPY79Wniwbr0Dy7LU=,tag:fa91EtRv3BJqVmjugQ4zyA==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1

terraform/wifi-289708231103/core-dumps-s3/main.tf

@@ -0,0 +1,59 @@
+provider "aws" {
+  region = var.aws_region
+}
+
+terraform {
+  required_version = ">= 1.0.0, < 2.0.0"
+
+  backend "s3" {
+    region         = "us-east-1"
+    bucket         = "tip-wifi-tfstate"
+    key            = "core-dumps-s3"
+    dynamodb_table = "terraform-state-lock"
+    encrypt        = true
+  }
+}
+
+locals {
+  common_tags = {
+    "ManagedBy" = "terraform"
+  }
+}
+
+resource "aws_s3_bucket" "openwifi-core-dumps" {
+  bucket = "openwifi-core-dumps"
+  tags   = local.common_tags
+}
+
+resource "aws_s3_bucket_acl" "openwifi-core-dumps" {
+  bucket = aws_s3_bucket.openwifi-core-dumps.id
+  acl    = "private"
+}
+
+resource "aws_iam_user" "openwifi-core-dump-handler" {
+  name = "openwifi-core-dump-handler"
+  tags = local.common_tags
+}
+
+resource "aws_iam_access_key" "openwifi-core-dump-handler" {
+  user = aws_iam_user.openwifi-core-dump-handler.name
+}
+
+resource "aws_iam_user_policy" "openwifi-core-dump-handler" {
+  name = "openwifi-core-dump-handler"
+  user = aws_iam_user.openwifi-core-dump-handler.name
+
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Action" : "s3:*",
+        "Resource" : [
+          aws_s3_bucket.openwifi-core-dumps.arn,
+          "${aws_s3_bucket.openwifi-core-dumps.arn}/*"
+        ]
+      }
+    ]
+  })
+}

terraform/wifi-289708231103/core-dumps-s3/terraform.tfvars

@@ -0,0 +1 @@
+aws_region = "us-east-1"

terraform/wifi-289708231103/core-dumps-s3/variables.tf

@@ -0,0 +1,4 @@
+variable "aws_region" {
+  description = "AWS region"
+  type        = string
+}