diff --git a/rust/Cargo.lock b/rust/Cargo.lock index d67933e4b..3ccfa2d30 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2473,6 +2473,9 @@ name = "firezone-telemetry" version = "0.1.0" dependencies = [ "anyhow", + "ip-packet", + "opentelemetry", + "opentelemetry_sdk", "parking_lot", "reqwest", "sentry", diff --git a/rust/connlib/tunnel/src/io.rs b/rust/connlib/tunnel/src/io.rs index 182195a23..ba1c9303a 100644 --- a/rust/connlib/tunnel/src/io.rs +++ b/rust/connlib/tunnel/src/io.rs @@ -3,7 +3,7 @@ mod nameserver_set; mod tcp_dns; mod udp_dns; -use crate::{device_channel::Device, dns, sockets::Sockets}; +use crate::{device_channel::Device, dns, otel, sockets::Sockets}; use anyhow::{Context as _, Result}; use firezone_logging::{telemetry_event, telemetry_span}; use futures::FutureExt as _; @@ -194,15 +194,15 @@ impl Io { self.packet_counter.add( num_ipv4 as u64, &[ - crate::otel::network_type_ipv4(), - crate::otel::network_io_direction_receive(), + otel::attr::network_type_ipv4(), + otel::attr::network_io_direction_receive(), ], ); self.packet_counter.add( num_ipv6 as u64, &[ - crate::otel::network_type_ipv6(), - crate::otel::network_io_direction_receive(), + otel::attr::network_type_ipv6(), + otel::attr::network_io_direction_receive(), ], ); @@ -312,8 +312,8 @@ impl Io { self.packet_counter.add( 1, &[ - crate::otel::network_type_for_packet(&packet), - crate::otel::network_io_direction_transmit(), + otel::attr::network_type_for_packet(&packet), + otel::attr::network_io_direction_transmit(), ], ); @@ -351,9 +351,9 @@ impl Io { self.packet_counter.add( 1, &[ - crate::otel::network_protocol_name(payload), - crate::otel::network_transport_udp(), - crate::otel::network_io_direction_transmit(), + otel::attr::network_protocol_name(payload), + otel::attr::network_transport_udp(), + otel::attr::network_io_direction_transmit(), ], ); } diff --git a/rust/connlib/tunnel/src/lib.rs b/rust/connlib/tunnel/src/lib.rs index aff71f203..7031f80ac 100644 --- a/rust/connlib/tunnel/src/lib.rs +++ b/rust/connlib/tunnel/src/lib.rs @@ -195,9 +195,9 @@ impl ClientTunnel { self.packet_counter.add( 1, &[ - crate::otel::network_protocol_name(received.packet), - crate::otel::network_transport_udp(), - crate::otel::network_io_direction_receive(), + otel::attr::network_protocol_name(received.packet), + otel::attr::network_transport_udp(), + otel::attr::network_io_direction_receive(), ], ); @@ -327,9 +327,9 @@ impl GatewayTunnel { self.packet_counter.add( 1, &[ - crate::otel::network_protocol_name(received.packet), - crate::otel::network_transport_udp(), - crate::otel::network_io_direction_receive(), + otel::attr::network_protocol_name(received.packet), + otel::attr::network_transport_udp(), + otel::attr::network_io_direction_receive(), ], ); diff --git a/rust/connlib/tunnel/src/otel.rs b/rust/connlib/tunnel/src/otel.rs index 7acd9bf29..df276e836 100644 --- a/rust/connlib/tunnel/src/otel.rs +++ b/rust/connlib/tunnel/src/otel.rs @@ -1,93 +1,23 @@ -use std::{io, net::SocketAddr}; +pub mod attr { + pub use firezone_telemetry::otel::attr::*; -use ip_packet::IpPacket; -use opentelemetry::{KeyValue, Value}; + use opentelemetry::KeyValue; -pub fn network_transport_udp() -> KeyValue { - KeyValue::new("network.transport", "udp") -} + pub fn network_protocol_name(payload: &[u8]) -> KeyValue { + const KEY: &str = "network.protocol.name"; -pub fn network_type_for_packet(p: &IpPacket) -> KeyValue { - match p { - IpPacket::Ipv4(_) => network_type_ipv4(), - IpPacket::Ipv6(_) => network_type_ipv6(), - } -} - -pub fn network_type_for_addr(addr: SocketAddr) -> KeyValue { - match addr { - SocketAddr::V4(_) => network_type_ipv4(), - SocketAddr::V6(_) => network_type_ipv6(), - } -} - -pub fn network_protocol_name(payload: &[u8]) -> KeyValue { - const KEY: &str = "network.protocol.name"; - - match payload { - [0..3, ..] => KeyValue::new(KEY, "stun"), - // Channel-data is a 4-byte header so the actual payload starts on the 5th byte - [64..=79, _, _, _, 0..3, ..] => KeyValue::new(KEY, "stun-over-turn"), - [64..=79, _, _, _, payload @ ..] if snownet::is_wireguard(payload) => { - KeyValue::new(KEY, "wireguard-over-turn") + match payload { + [0..3, ..] => KeyValue::new(KEY, "stun"), + // Channel-data is a 4-byte header so the actual payload starts on the 5th byte + [64..=79, _, _, _, 0..3, ..] => KeyValue::new(KEY, "stun-over-turn"), + [64..=79, _, _, _, payload @ ..] if snownet::is_wireguard(payload) => { + KeyValue::new(KEY, "wireguard-over-turn") + } + [64..=79, _, _, _, ..] => KeyValue::new(KEY, "unknown-over-turn"), + payload if snownet::is_wireguard(payload) => KeyValue::new(KEY, "wireguard"), + _ => KeyValue::new(KEY, "unknown"), } - [64..=79, _, _, _, ..] => KeyValue::new(KEY, "unknown-over-turn"), - payload if snownet::is_wireguard(payload) => KeyValue::new(KEY, "wireguard"), - _ => KeyValue::new(KEY, "unknown"), } } -pub fn network_type_ipv4() -> KeyValue { - KeyValue::new("network.type", "ipv4") -} - -pub fn network_type_ipv6() -> KeyValue { - KeyValue::new("network.type", "ipv6") -} - -pub fn network_io_direction_receive() -> KeyValue { - KeyValue::new("network.io.direction", "receive") -} - -pub fn network_io_direction_transmit() -> KeyValue { - KeyValue::new("network.io.direction", "transmit") -} - -pub fn io_error_code(e: &io::Error) -> KeyValue { - KeyValue::new("error.code", e.raw_os_error().unwrap_or_default() as i64) -} - -pub fn io_error_type(e: &io::Error) -> KeyValue { - error_type(format!("io::ErrorKind::{:?}", e.kind())) -} - -pub fn error_type(ty: impl Into) -> KeyValue { - KeyValue::new("error.type", ty) -} - -pub mod metrics { - use opentelemetry::metrics::Counter; - - pub fn network_packet_dropped() -> Counter { - opentelemetry::global::meter("connlib") - .u64_counter("network.packet.dropped") - .with_description("Count of packets that are dropped or discarded") - .with_unit("{packet}") - .init() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn io_error_type_serialisation() { - let error = io::Error::from(io::ErrorKind::NetworkUnreachable); - - assert_eq!( - io_error_type(&error), - KeyValue::new("error.type", "io::ErrorKind::NetworkUnreachable") - ); - } -} +pub use firezone_telemetry::otel::metrics; diff --git a/rust/connlib/tunnel/src/peer.rs b/rust/connlib/tunnel/src/peer.rs index b32c84797..7ef595314 100644 --- a/rust/connlib/tunnel/src/peer.rs +++ b/rust/connlib/tunnel/src/peer.rs @@ -15,7 +15,7 @@ use ip_network_table::IpNetworkTable; use ip_packet::{IpPacket, PacketBuilder, Protocol, UnsupportedProtocol, icmpv4, icmpv6}; use crate::utils::network_contains_network; -use crate::{GatewayEvent, IpConfig}; +use crate::{GatewayEvent, IpConfig, otel}; use anyhow::{Context, Result, bail}; use nat_table::{NatTable, TranslateIncomingResult}; @@ -95,7 +95,7 @@ impl ClientOnGateway { nat_table: Default::default(), buffered_events: Default::default(), internet_resource_enabled: false, - num_dropped_packets: crate::otel::metrics::network_packet_dropped(), + num_dropped_packets: otel::metrics::network_packet_dropped(), } } @@ -364,9 +364,9 @@ impl ClientOnGateway { self.num_dropped_packets.add( 1, &[ - crate::otel::network_type_for_packet(&packet), - crate::otel::network_io_direction_receive(), - crate::otel::error_type(e.root_cause().to_string()), + otel::attr::network_type_for_packet(&packet), + otel::attr::network_io_direction_receive(), + otel::attr::error_type(e.root_cause().to_string()), ], ); @@ -401,9 +401,9 @@ impl ClientOnGateway { self.num_dropped_packets.add( 1, &[ - crate::otel::network_type_for_packet(&packet), - crate::otel::network_io_direction_receive(), - crate::otel::error_type("ExpiredNatSession"), + otel::attr::network_type_for_packet(&packet), + otel::attr::network_io_direction_receive(), + otel::attr::error_type("ExpiredNatSession"), ], ); diff --git a/rust/connlib/tunnel/src/sockets.rs b/rust/connlib/tunnel/src/sockets.rs index ec9ca6633..0fd148614 100644 --- a/rust/connlib/tunnel/src/sockets.rs +++ b/rust/connlib/tunnel/src/sockets.rs @@ -1,3 +1,4 @@ +use crate::otel; use anyhow::Result; use futures::{SinkExt, StreamExt, ready}; use gat_lending_iterator::LendingIterator; @@ -201,10 +202,10 @@ impl ThreadedUdpSocket { if let Err(e) = socket.send(datagram).await { if let Some(io) = e.downcast_ref::() { io_error_counter.add(1, &[ - crate::otel::network_io_direction_transmit(), - crate::otel::network_type_for_addr(addr), - crate::otel::io_error_type(io), - crate::otel::io_error_code(io) + otel::attr::network_io_direction_transmit(), + otel::attr::network_type_for_addr(addr), + otel::attr::io_error_type(io), + otel::attr::io_error_code(io) ]); } @@ -226,10 +227,10 @@ impl ThreadedUdpSocket { if let Some(io) = result.as_ref().err().and_then(|e| e.downcast_ref::()) { io_error_counter.add(1, &[ - crate::otel::network_io_direction_receive(), - crate::otel::network_type_for_addr(addr), - crate::otel::io_error_type(io), - crate::otel::io_error_code(io) + otel::attr::network_io_direction_receive(), + otel::attr::network_type_for_addr(addr), + otel::attr::io_error_type(io), + otel::attr::io_error_code(io) ]); } diff --git a/rust/connlib/tunnel/src/unique_packet_buffer.rs b/rust/connlib/tunnel/src/unique_packet_buffer.rs index f3a50ea6c..1ef587548 100644 --- a/rust/connlib/tunnel/src/unique_packet_buffer.rs +++ b/rust/connlib/tunnel/src/unique_packet_buffer.rs @@ -2,6 +2,8 @@ use ip_packet::IpPacket; use opentelemetry::KeyValue; use ringbuffer::{AllocRingBuffer, RingBuffer}; +use crate::otel; + pub struct UniquePacketBuffer { buffer: AllocRingBuffer, tag: &'static str, @@ -41,10 +43,10 @@ impl UniquePacketBuffer { self.num_dropped_packets.add( 1, &[ - crate::otel::network_type_for_packet(&new), - crate::otel::network_io_direction_transmit(), + otel::attr::network_type_for_packet(&new), + otel::attr::network_io_direction_transmit(), KeyValue::new("system.buffer.pool.name", self.tag), - crate::otel::error_type("BufferFull"), + otel::attr::error_type("BufferFull"), ], ); } diff --git a/rust/gateway/src/main.rs b/rust/gateway/src/main.rs index c527330b9..6f045e9fc 100644 --- a/rust/gateway/src/main.rs +++ b/rust/gateway/src/main.rs @@ -11,7 +11,7 @@ use firezone_bin_shared::{ platform::{tcp_socket_factory, udp_socket_factory}, }; -use firezone_telemetry::Telemetry; +use firezone_telemetry::{Telemetry, otel}; use firezone_tunnel::GatewayTunnel; use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use phoenix_channel::LoginUrl; @@ -105,18 +105,25 @@ async fn try_main(cli: Cli) -> Result { firezone_logging::setup_global_subscriber(layer::Identity::default()) .context("Failed to set up logging")?; - if cli.metrics { - let exporter = opentelemetry_stdout::MetricsExporter::default(); - let reader = PeriodicReader::builder(exporter, opentelemetry_sdk::runtime::Tokio).build(); - let provider = SdkMeterProvider::builder().with_reader(reader).build(); - - opentelemetry::global::set_meter_provider(provider); - } - let firezone_id = get_firezone_id(cli.firezone_id).await .context("Couldn't read FIREZONE_ID or write it to disk: Please provide it through the env variable or provide rw access to /var/lib/firezone/")?; Telemetry::set_firezone_id(firezone_id.clone()); + if cli.metrics { + let exporter = opentelemetry_stdout::MetricsExporter::default(); + let reader = PeriodicReader::builder(exporter, opentelemetry_sdk::runtime::Tokio).build(); + let provider = SdkMeterProvider::builder() + .with_reader(reader) + .with_resource(otel::default_resource_with([ + otel::attr::service_name!(), + otel::attr::service_version!(), + otel::attr::service_instance_id(firezone_id.clone()), + ])) + .build(); + + opentelemetry::global::set_meter_provider(provider); + } + let login = LoginUrl::gateway( cli.api_url, &SecretString::new(cli.token), diff --git a/rust/headless-client/src/main.rs b/rust/headless-client/src/main.rs index 7f9fecf88..34cdfa878 100644 --- a/rust/headless-client/src/main.rs +++ b/rust/headless-client/src/main.rs @@ -15,6 +15,7 @@ use firezone_headless_client::{ }; use firezone_logging::telemetry_span; use firezone_telemetry::Telemetry; +use firezone_telemetry::otel; use futures::StreamExt as _; use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use phoenix_channel::LoginUrl; @@ -192,7 +193,7 @@ fn main() -> Result<()> { let url = LoginUrl::client( cli.api_url, &token, - firezone_id, + firezone_id.clone(), cli.firezone_name, device_id::device_info(), )?; @@ -213,7 +214,14 @@ fn main() -> Result<()> { let exporter = opentelemetry_stdout::MetricsExporter::default(); let reader = PeriodicReader::builder(exporter, opentelemetry_sdk::runtime::Tokio).build(); - let provider = SdkMeterProvider::builder().with_reader(reader).build(); + let provider = SdkMeterProvider::builder() + .with_reader(reader) + .with_resource(otel::default_resource_with([ + otel::attr::service_name!(), + otel::attr::service_version!(), + otel::attr::service_instance_id(firezone_id), + ])) + .build(); opentelemetry::global::set_meter_provider(provider); } diff --git a/rust/telemetry/Cargo.toml b/rust/telemetry/Cargo.toml index 40c3da34c..f90eabaa6 100644 --- a/rust/telemetry/Cargo.toml +++ b/rust/telemetry/Cargo.toml @@ -6,6 +6,9 @@ license = { workspace = true } [dependencies] anyhow = { workspace = true } +ip-packet = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } parking_lot = { workspace = true } reqwest = { workspace = true } sentry = { workspace = true, features = ["contexts", "backtrace", "debug-images", "panic", "reqwest", "rustls", "tracing"] } diff --git a/rust/telemetry/src/lib.rs b/rust/telemetry/src/lib.rs index 56589ef27..a99528ba6 100644 --- a/rust/telemetry/src/lib.rs +++ b/rust/telemetry/src/lib.rs @@ -6,6 +6,7 @@ use env::ON_PREM; use sentry::protocol::SessionStatus; pub mod feature_flags; +pub mod otel; pub struct Dsn(&'static str); diff --git a/rust/telemetry/src/otel.rs b/rust/telemetry/src/otel.rs new file mode 100644 index 000000000..77a61ee9b --- /dev/null +++ b/rust/telemetry/src/otel.rs @@ -0,0 +1,128 @@ +use std::time::Duration; + +use opentelemetry::KeyValue; +use opentelemetry_sdk::{ + Resource, + resource::{ResourceDetector, TelemetryResourceDetector}, +}; + +pub mod attr { + use ip_packet::IpPacket; + use opentelemetry::Value; + use std::{io, net::SocketAddr}; + + use super::*; + + #[macro_export] + macro_rules! service_name { + () => { + ::opentelemetry::KeyValue::new("service.name", env!("CARGO_PKG_NAME")) + }; + } + + #[macro_export] + macro_rules! service_version { + () => { + ::opentelemetry::KeyValue::new("service.version", env!("CARGO_PKG_VERSION")) + }; + } + + pub use service_name; + pub use service_version; + + pub fn service_instance_id(firezone_id: String) -> KeyValue { + KeyValue::new("service.instance.id", firezone_id) + } + + pub fn network_transport_udp() -> KeyValue { + KeyValue::new("network.transport", "udp") + } + + pub fn network_type_for_packet(p: &IpPacket) -> KeyValue { + match p { + IpPacket::Ipv4(_) => network_type_ipv4(), + IpPacket::Ipv6(_) => network_type_ipv6(), + } + } + + pub fn network_type_for_addr(addr: SocketAddr) -> KeyValue { + match addr { + SocketAddr::V4(_) => network_type_ipv4(), + SocketAddr::V6(_) => network_type_ipv6(), + } + } + + pub fn network_type_ipv4() -> KeyValue { + KeyValue::new("network.type", "ipv4") + } + + pub fn network_type_ipv6() -> KeyValue { + KeyValue::new("network.type", "ipv6") + } + + pub fn network_io_direction_receive() -> KeyValue { + KeyValue::new("network.io.direction", "receive") + } + + pub fn network_io_direction_transmit() -> KeyValue { + KeyValue::new("network.io.direction", "transmit") + } + + pub fn io_error_code(e: &io::Error) -> KeyValue { + KeyValue::new("error.code", e.raw_os_error().unwrap_or_default() as i64) + } + + pub fn io_error_type(e: &io::Error) -> KeyValue { + error_type(format!("io::ErrorKind::{:?}", e.kind())) + } + + pub fn error_type(value: impl Into) -> KeyValue { + KeyValue::new("error.type", value) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn io_error_type_serialisation() { + let error = io::Error::from(io::ErrorKind::NetworkUnreachable); + + assert_eq!( + io_error_type(&error), + KeyValue::new("error.type", "io::ErrorKind::NetworkUnreachable") + ); + } + } +} + +pub mod metrics { + use opentelemetry::metrics::Counter; + + pub fn network_packet_dropped() -> Counter { + opentelemetry::global::meter("connlib") + .u64_counter("network.packet.dropped") + .with_description("Count of packets that are dropped or discarded") + .with_unit("{packet}") + .init() + } +} + +pub fn default_resource_with(attributes: [KeyValue; N]) -> Resource { + Resource::from_detectors( + Duration::from_secs(0), + vec![ + Box::new(TelemetryResourceDetector), + Box::new(OsResourceDetector), + ], + ) + .merge(&Resource::new(attributes)) +} + +pub struct OsResourceDetector; + +impl ResourceDetector for OsResourceDetector { + fn detect(&self, _: Duration) -> Resource { + Resource::new([KeyValue::new("os.type", std::env::consts::OS)]) + } +}