feat(telemetry): pre-resolve PostHog ingest host (#10207)

In order to effectively share the HTTP client for requests to PostHog,
we pre-resolve the IPs of the host and create a lazily initialised
`reqwest::Client` that gets shared between all analytics calls.

rust/telemetry/src/analytics.rs

@@ -4,7 +4,7 @@ use anyhow::{Context as _, Result, bail};
 use serde::Serialize;
 use sha2::Digest as _;
 
-use crate::{ApiUrl, Env, Telemetry, posthog::RUNTIME};
+use crate::{ApiUrl, Env, Telemetry, posthog};
 
 /// Records a `new_session` event for a particular user and API url.
 ///
@@ -16,7 +16,7 @@ pub fn new_session(maybe_legacy_id: String, api_url: String) {
         maybe_legacy_id
     };
 
-    RUNTIME.spawn(async move {
+    posthog::RUNTIME.spawn(async move {
         if let Err(e) = capture(
             "new_session",
             distinct_id,
@@ -43,7 +43,7 @@ pub fn identify(release: String, account_slug: Option<String>) {
         return;
     };
 
-    RUNTIME.spawn({
+    posthog::RUNTIME.spawn({
         async move {
             if let Err(e) = capture(
                 "$identify",
@@ -76,7 +76,7 @@ pub fn feature_flag_called(name: impl Into<String>) {
     };
     let feature_flag = name.into();
 
-    RUNTIME.spawn({
+    posthog::RUNTIME.spawn({
         async move {
             if let Err(e) = capture(
                 "$feature_flag_called",
@@ -113,10 +113,9 @@ where
         return Ok(());
     };
 
-    let response = reqwest::ClientBuilder::new()
-        .connection_verbose(true)
-        .build()?
-        .post("https://us.i.posthog.com/i/v0/e/")
+    let response = posthog::CLIENT
+        .as_ref()?
+        .post(format!("https://{}/i/v0/e/", posthog::INGEST_HOST))
         .json(&CaptureRequest {
             api_key: api_key.to_string(),
             distinct_id,

rust/telemetry/src/feature_flags.rs

@@ -15,10 +15,7 @@ use sha2::Digest as _;
 use tracing::{Metadata, level_filters::LevelFilter};
 use tracing_subscriber::filter::Targets;
 
-use crate::{
-    Env,
-    posthog::{POSTHOG_API_KEY_PROD, POSTHOG_API_KEY_STAGING, RUNTIME},
-};
+use crate::{Env, posthog};
 
 pub(crate) const RE_EVAL_DURATION: Duration = Duration::from_secs(5 * 60);
 
@@ -53,8 +50,8 @@ pub(crate) async fn evaluate_now(user_id: String, env: Env) {
     }
 
     let api_key = match env {
-        Env::Production => POSTHOG_API_KEY_PROD,
-        Env::Staging => POSTHOG_API_KEY_STAGING,
+        Env::Production => posthog::API_KEY_PROD,
+        Env::Staging => posthog::API_KEY_STAGING,
         Env::OnPrem | Env::DockerCompose | Env::Localhost => return,
     };
 
@@ -77,7 +74,7 @@ pub(crate) fn reevaluate(user_id: String, env: &str) {
         return;
     };
 
-    RUNTIME.spawn(evaluate_now(user_id, env));
+    posthog::RUNTIME.spawn(evaluate_now(user_id, env));
 }
 
 pub(crate) async fn reeval_timer() {
@@ -112,12 +109,9 @@ async fn decide(
         maybe_legacy_id
     };
 
-    let response = reqwest::ClientBuilder::new()
-        .connection_verbose(true)
-        .pool_idle_timeout(RE_EVAL_DURATION * 2) // Ensure we reuse the same connection if possible.
-        .pool_max_idle_per_host(1)
-        .build()?
-        .post("https://us.i.posthog.com/decide?v=3")
+    let response = posthog::CLIENT
+        .as_ref()?
+        .post(format!("https://{}/decide?v=3", posthog::INGEST_HOST))
         .json(&DecideRequest {
             api_key,
             distinct_id,

rust/telemetry/src/posthog.rs

@@ -1,19 +1,22 @@
-use std::sync::LazyLock;
+use std::{net::ToSocketAddrs as _, sync::LazyLock, time::Duration};
 use tokio::runtime::Runtime;
 
 use crate::Env;
 
-pub(crate) const POSTHOG_API_KEY_PROD: &str = "phc_uXXl56plyvIBHj81WwXBLtdPElIRbm7keRTdUCmk8ll";
-pub(crate) const POSTHOG_API_KEY_STAGING: &str = "phc_tHOVtq183RpfKmzadJb4bxNpLM5jzeeb1Gu8YSH3nsK";
-pub(crate) const POSTHOG_API_KEY_ON_PREM: &str = "phc_4R9Ii6q4SEofVkH7LvajwuJ3nsGFhCj0ZlfysS2FNc";
+pub(crate) const API_KEY_PROD: &str = "phc_uXXl56plyvIBHj81WwXBLtdPElIRbm7keRTdUCmk8ll";
+pub(crate) const API_KEY_STAGING: &str = "phc_tHOVtq183RpfKmzadJb4bxNpLM5jzeeb1Gu8YSH3nsK";
+pub(crate) const API_KEY_ON_PREM: &str = "phc_4R9Ii6q4SEofVkH7LvajwuJ3nsGFhCj0ZlfysS2FNc";
 
 pub(crate) static RUNTIME: LazyLock<Runtime> = LazyLock::new(init_runtime);
+pub(crate) static CLIENT: LazyLock<reqwest::Result<reqwest::Client>> = LazyLock::new(init_client);
+
+pub(crate) const INGEST_HOST: &str = "us.i.posthog.com";
 
 pub(crate) fn api_key_for_env(env: Env) -> Option<&'static str> {
     match env {
-        Env::Production => Some(POSTHOG_API_KEY_PROD),
-        Env::Staging => Some(POSTHOG_API_KEY_STAGING),
-        Env::OnPrem => Some(POSTHOG_API_KEY_ON_PREM),
+        Env::Production => Some(API_KEY_PROD),
+        Env::Staging => Some(API_KEY_STAGING),
+        Env::OnPrem => Some(API_KEY_ON_PREM),
         Env::DockerCompose | Env::Localhost => None,
     }
 }
@@ -32,3 +35,26 @@ fn init_runtime() -> Runtime {
 
     runtime
 }
+
+/// Initialize the client to use for evaluating feature flags.
+fn init_client() -> reqwest::Result<reqwest::Client> {
+    let ingest_host_addresses = (INGEST_HOST, 443u16)
+        .to_socket_addrs()
+        .inspect_err(|e| {
+            tracing::error!("Failed to resolve ingest host (`{INGEST_HOST}`) IPs: {e:#}")
+        })
+        .unwrap_or_default()
+        .collect::<Vec<_>>();
+
+    tracing::debug!(host = %INGEST_HOST, addresses = ?ingest_host_addresses, "Resolved PostHog ingest host addresses");
+
+    reqwest::ClientBuilder::new()
+        .connection_verbose(true)
+        .pool_idle_timeout(None) // Never remove idle connections.
+        .pool_max_idle_per_host(1)
+        .http2_prior_knowledge() // We know PostHog supports HTTP/2.
+        .http2_keep_alive_timeout(Duration::from_secs(1))
+        .http2_keep_alive_interval(Duration::from_secs(5)) // Use keep-alive to detect broken connections.
+        .resolve_to_addrs(INGEST_HOST, &ingest_host_addresses)
+        .build()
+}