From d46ce9ab94ab5dc0b726fc9f5afa545016aca6ad Mon Sep 17 00:00:00 2001 From: Thomas Eizinger Date: Sat, 8 Mar 2025 13:07:46 +1100 Subject: [PATCH] chore(connlib): setup feature-flag infrastructure (#8382) In order to more safely roll out certain changes, being able to runtime-toggle features is crucial. For this purpose, we build a simple integration with Posthog that allows us to evaluate feature flags based on the Firezone ID of a Client or Gateway. The feature flags are also set in a dedicated context for Sentry events. This allows us to see, which feature flags were active when a certain error is logged to Sentry. --- rust/Cargo.lock | 5 ++ rust/Cargo.toml | 1 + rust/telemetry/Cargo.toml | 5 ++ rust/telemetry/src/lib.rs | 99 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 107 insertions(+), 3 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0af838334..48efc1427 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2249,7 +2249,12 @@ dependencies = [ name = "firezone-telemetry" version = "0.1.0" dependencies = [ + "anyhow", + "parking_lot", + "reqwest", "sentry", + "serde", + "serde_json", "thiserror 1.0.69", "tokio", "tracing", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 293e2c3b7..0adbe9eb8 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -86,6 +86,7 @@ os_info = { version = "3", default-features = false } output_vt100 = "0.1" png = "0.17.16" proptest = "1.6.0" +parking_lot = "0.12.3" proptest-state-machine = "0.3.1" quinn-udp = { version = "0.5.8", features = ["fast-apple-datapath"] } rand = "0.8.5" diff --git a/rust/telemetry/Cargo.toml b/rust/telemetry/Cargo.toml index 68ae4c714..8788fd8b8 100644 --- a/rust/telemetry/Cargo.toml +++ b/rust/telemetry/Cargo.toml @@ -5,7 +5,12 @@ edition = { workspace = true } license = { workspace = true } [dependencies] +anyhow = { workspace = true } +parking_lot = { workspace = true } +reqwest = { workspace = true, features = ["blocking"] } sentry = { workspace = true, features = ["contexts", "backtrace", "debug-images", "panic", "reqwest", "rustls", "tracing"] } +serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true, features = ["rt"] } tracing = { workspace = true } diff --git a/rust/telemetry/src/lib.rs b/rust/telemetry/src/lib.rs index 26181607b..6f866da00 100644 --- a/rust/telemetry/src/lib.rs +++ b/rust/telemetry/src/lib.rs @@ -1,7 +1,15 @@ -use std::{sync::Arc, time::Duration}; +#![cfg_attr(test, allow(clippy::unwrap_used))] +use std::{ + sync::{Arc, LazyLock}, + time::Duration, +}; + +use anyhow::{Context, Result}; use env::ON_PREM; +use parking_lot::Mutex; use sentry::protocol::SessionStatus; +use serde::{Deserialize, Serialize}; pub struct Dsn(&'static str); @@ -18,6 +26,24 @@ pub const HEADLESS_DSN: Dsn = Dsn("https://bc27dca8bb37be0142c48c4f89647c13@o450 pub const RELAY_DSN: Dsn = Dsn("https://9d5f664d8f8f7f1716d4b63a58bcafd5@o4507971108339712.ingest.us.sentry.io/4508373298970624"); pub const TESTING: Dsn = Dsn("https://55ef451fca9054179a11f5d132c02f45@o4507971108339712.ingest.us.sentry.io/4508792604852224"); +const POSTHOG_API_KEY: &str = "phc_uXXl56plyvIBHj81WwXBLtdPElIRbm7keRTdUCmk8ll"; + +// Process-wide storage of enabled feature flags. +// +// Defaults to everything off. +static FEATURE_FLAGS: LazyLock> = LazyLock::new(Mutex::default); + +/// Exposes all feature flags as public, static functions. +/// +/// These only ever hit an in-memory location so can even be called from hot paths. +pub mod feature_flags { + use crate::*; + + pub fn icmp_unreachable_instead_of_nat64() -> bool { + FEATURE_FLAGS.lock().icmp_unreachable_instead_of_nat64 + } +} + mod env { use std::borrow::Cow; @@ -150,8 +176,23 @@ impl Telemetry { } pub fn set_firezone_id(id: String) { - update_user(|user| { - user.id = Some(id); + update_user({ + let id = id.clone(); + |user| user.id = Some(id) + }); + + std::thread::spawn(|| { + let flags = evaluate_feature_flags(id) + .inspect_err(|e| tracing::debug!("Failed to evaluate feature flags: {e:#}")) + .unwrap_or_default(); + + tracing::debug!(?flags, "Evaluated feature-flags"); + + *FEATURE_FLAGS.lock() = flags; + + sentry::Hub::main().configure_scope(|scope| { + scope.set_context("flags", sentry_flag_context(flags)); + }); }); } } @@ -169,6 +210,58 @@ fn set_current_user(user: Option) { sentry::Hub::main().configure_scope(|scope| scope.set_user(user)); } +fn evaluate_feature_flags(id: String) -> Result { + let json = reqwest::blocking::Client::new() + .post("https://us.i.posthog.com/decide?v=3") + .body(format!( + r#"{{ + "api_key": "{POSTHOG_API_KEY}", + "distinct_id": "{id}" + }}"# + )) + .send() + .context("Failed to send POST request")? + .json::() + .context("Failed to deserialize response")?; + + Ok(json.feature_flags) +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct DecideResponse { + feature_flags: FeatureFlags, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy)] +struct FeatureFlags { + #[serde(default)] + icmp_unreachable_instead_of_nat64: bool, +} + +fn sentry_flag_context(flags: FeatureFlags) -> sentry::protocol::Context { + #[derive(Debug, serde::Serialize)] + #[serde(tag = "flag", rename_all = "snake_case")] + enum SentryFlag { + IcmpUnreachableInsteadOfNat64 { result: bool }, + } + + // Exhaustive destruction so we don't forget to update this when we add a flag. + let FeatureFlags { + icmp_unreachable_instead_of_nat64, + } = flags; + + let value = serde_json::json!({ + "values": [ + SentryFlag::IcmpUnreachableInsteadOfNat64 { + result: icmp_unreachable_instead_of_nat64, + } + ] + }); + + sentry::protocol::Context::Other(serde_json::from_value(value).expect("to and from json works")) +} + #[cfg(test)] mod tests { use super::*;