Deploy OTEL collectors to every instance

terraform/modules/elixir-app/main.tf

@@ -114,6 +114,10 @@ resource "google_project_iam_member" "cloudtrace" {
 }
 
 # Deploy the app
+data "template_file" "clout-init" {
+  template = file("${path.module}/templates/cloud-init.yaml")
+}
+
 resource "google_compute_instance_template" "application" {
   project = var.project_id
 
@@ -175,7 +179,7 @@ resource "google_compute_instance_template" "application" {
     enable_vtpm                 = true
   }
 
-  metadata = merge({
+  metadata = {
     gce-container-declaration = yamlencode({
       spec = {
         containers = [{
@@ -190,6 +194,8 @@ resource "google_compute_instance_template" "application" {
       }
     })
 
+    user-data = data.template_file.clout-init.rendered
+
     google-logging-enabled = "true"
     # Enable FluentBit agent for logging, which will be default one from COS 109
     # Re-enable once https://issuetracker.google.com/issues/285950891 is closed
@@ -197,7 +203,7 @@ resource "google_compute_instance_template" "application" {
 
     # Report health-related metrics to Cloud Monitoring
     google-monitoring-enabled = "true"
-  })
+  }
 
   depends_on = [
     google_project_service.compute,

terraform/modules/elixir-app/templates/cloud-init.yaml

@@ -0,0 +1,62 @@
+#cloud-config
+
+users:
+  - name: cloudservice
+    uid: 2000
+
+write_files:
+  - path: /etc/otelcol-contrib/config.yaml
+    permissions: 0644
+    owner: root
+    content: |
+      receivers:
+        otlp:
+          protocols:
+            grpc:
+            http:
+      exporters:
+        googlecloud:
+          log:
+            default_log_name: opentelemetry.io/collector-exported-log
+      processors:
+        memory_limiter:
+          check_interval: 1s
+          limit_percentage: 65
+          spike_limit_percentage: 20
+        batch:
+        resourcedetection:
+          detectors: [gcp]
+          timeout: 10s
+      service:
+        pipelines:
+          traces:
+            receivers: [otlp]
+            processors: [memory_limiter, batch]
+            exporters: [googlecloud]
+          metrics:
+            receivers: [otlp]
+            processors: [memory_limiter, batch]
+            exporters: [googlecloud]
+          logs:
+            receivers: [otlp]
+            processors: [memory_limiter, batch]
+            exporters: [googlecloud]
+
+  - path: /etc/systemd/system/otel-collector.service
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Start an OpenTelemetry collector docker container
+
+      [Service]
+      TimeoutStartSec=0
+      Restart=always
+      ExecStartPre=/usr/bin/docker pull otel/opentelemetry-collector-contrib:0.85.0
+      ExecStart=/usr/bin/docker run --rm -u 2000 --name=otel-collector --network host --volume /etc/otelcol-contrib/:/etc/otelcol-contrib/ otel/opentelemetry-collector-contrib:0.85.0
+      ExecStop=/usr/bin/docker stop otel-collector
+      ExecStopPost=/usr/bin/docker rm otel-collector
+
+runcmd:
+  - systemctl daemon-reload
+  - systemctl start otel-collector.service

terraform/modules/relay-app/main.tf

@@ -22,6 +22,10 @@ locals {
       name  = "RUST_LOG"
       value = var.observability_log_level
     },
+    {
+      name  = "RUST_BACKTRACE"
+      value = "full"
+    },
     {
       name  = "LOG_FORMAT"
       value = "google-cloud"

terraform/modules/relay-app/templates/cloud-init.yaml

@@ -5,7 +5,7 @@ users:
     uid: 2000
 
 write_files:
-  - path: /etc/otel/config.yaml
+  - path: /etc/otelcol-contrib/config.yaml
     permissions: 0644
     owner: root
     content: |
@@ -40,7 +40,7 @@ write_files:
           logs:
             receivers: [otlp]
             processors: [memory_limiter, batch]
-            exporters: []
+            exporters: [googlecloud]
 
   - path: /etc/systemd/system/otel-collector.service
     permissions: 0644
@@ -52,8 +52,8 @@ write_files:
       [Service]
       TimeoutStartSec=0
       Restart=always
-      ExecStartPre=/usr/bin/docker pull otel/opentelemetry-collector-contrib:0.84.0
-      ExecStart=/usr/bin/docker run --rm -u 2000 --name=otel-collector --expose 4317 --expose 55681 otel/opentelemetry-collector-contrib:0.84.0
+      ExecStartPre=/usr/bin/docker pull otel/opentelemetry-collector-contrib:0.85.0
+      ExecStart=/usr/bin/docker run --rm -u 2000 --name=otel-collector --network host --volume /etc/otelcol-contrib/:/etc/otelcol-contrib/ otel/opentelemetry-collector-contrib:0.85.0
       ExecStop=/usr/bin/docker stop otel-collector
       ExecStopPost=/usr/bin/docker rm otel-collector