refactor(portal): Update elixir TF module (#9410)

Why:

* In order to reduce the number of traces/spans being sent to GCP a
  custom otel-collector config is needed for each type of node in our
  portal deployment.  This commit allows the elixir TF module to accept
  a otel-collector config at the time of use, rather than being hard
  coded in to the module itself.

terraform/modules/google-cloud/apps/elixir/main.tf

@@ -194,7 +194,8 @@ resource "google_compute_instance_template" "application" {
     })
 
     user-data = templatefile("${path.module}/templates/cloud-init.yaml", {
-      swap_size_gb = var.compute_swap_size_gb
+      swap_size_gb        = var.compute_swap_size_gb,
+      otel_config_content = indent(6, var.otel_config)
     })
 
     google-logging-enabled       = "true"

terraform/modules/google-cloud/apps/elixir/templates/cloud-init.yaml

@@ -5,43 +5,7 @@ write_files:
     permissions: "0644"
     owner: root
     content: |
-      receivers:
-        otlp:
-          protocols:
-            http:
-              endpoint: localhost:4318
-      exporters:
-        googlecloud:
-          log:
-            default_log_name: opentelemetry.io/collector-exported-log
-      processors:
-        memory_limiter:
-          check_interval: 1s
-          limit_percentage: 65
-          spike_limit_percentage: 20
-        batch:
-        resourcedetection:
-          detectors: [gcp]
-          timeout: 10s
-        filter/drophttp:
-          error_mode: ignore
-          traces:
-            span:
-              - 'attributes["http.target"] == "/healthz"'
-      service:
-        pipelines:
-          traces:
-            receivers: [otlp]
-            processors: [memory_limiter, filter/drophttp, batch]
-            exporters: [googlecloud]
-          metrics:
-            receivers: [otlp]
-            processors: [memory_limiter, filter/drophttp, batch]
-            exporters: [googlecloud]
-          logs:
-            receivers: [otlp]
-            processors: [memory_limiter, filter/drophttp, batch]
-            exporters: [googlecloud]
+      ${otel_config_content}
 
   - path: /etc/systemd/system/otel-collector.service
     permissions: "0644"
@@ -53,8 +17,8 @@ write_files:
       [Service]
       TimeoutStartSec=0
       Restart=always
-      ExecStartPre=/usr/bin/docker pull otel/opentelemetry-collector-contrib:0.119.0
-      ExecStart=/usr/bin/docker run --rm -u 2000 --name=otel-collector --network host --volume /etc/otelcol-contrib/:/etc/otelcol-contrib/ otel/opentelemetry-collector-contrib:0.119.0
+      ExecStartPre=/usr/bin/docker pull otel/opentelemetry-collector-contrib:0.127.0
+      ExecStart=/usr/bin/docker run --rm -u 2000 --name=otel-collector --network host --volume /etc/otelcol-contrib/:/etc/otelcol-contrib/ otel/opentelemetry-collector-contrib:0.127.0
       ExecStop=/usr/bin/docker stop otel-collector
       ExecStopPost=/usr/bin/docker rm otel-collector
 

terraform/modules/google-cloud/apps/elixir/variables.tf

@@ -152,6 +152,10 @@ variable "observability_log_level" {
   description = "Sets LOG_LEVEL environment variable which applications should use to configure Elixir Logger. Default: 'info'."
 }
 
+variable "otel_config" {
+  type        = string
+  description = "otel-collector YAML config content"
+}
 
 ################################################################################
 ## Erlang