diff --git a/rust/gui-client/src-tauri/src/client/gui.rs b/rust/gui-client/src-tauri/src/client/gui.rs
index 83c4c6c2f..6d994ebb9 100644
--- a/rust/gui-client/src-tauri/src/client/gui.rs
+++ b/rust/gui-client/src-tauri/src/client/gui.rs
@@ -22,6 +22,7 @@ use url::Url;
 
 use ControllerRequest as Req;
 
+mod ran_before;
 mod system_tray_menu;
 
 #[cfg(target_os = "linux")]
@@ -499,10 +500,7 @@ impl Controller {
         };
 
         let api_url = self.advanced_settings.api_url.clone();
-        tracing::info!(
-            api_url = api_url.to_string(),
-            "Calling connlib Session::connect"
-        );
+        tracing::info!(api_url = api_url.to_string(), "Starting connlib...");
 
         let mut connlib = ipc::Client::connect(
             api_url.as_str(),
@@ -522,6 +520,7 @@ impl Controller {
         });
         self.refresh_system_tray_menu()?;
 
+        ran_before::set().await?;
         Ok(())
     }
 
@@ -741,21 +740,9 @@ async fn run_controller(
     logging_handles: client::logging::Handles,
     advanced_settings: AdvancedSettings,
 ) -> Result<()> {
-    let session_dir =
-        firezone_headless_client::known_dirs::session().context("Couldn't find session dir")?;
-    let ran_before_path = session_dir.join("ran_before.txt");
-    if !tokio::fs::try_exists(&ran_before_path).await? {
-        let win = app
-            .get_window("welcome")
-            .context("Couldn't get handle to Welcome window")?;
-        win.show()?;
-        tokio::fs::create_dir_all(&session_dir).await?;
-        tokio::fs::write(&ran_before_path, &[]).await?;
-    }
-
     let mut controller = Controller {
         advanced_settings,
-        app,
+        app: app.clone(),
         auth: client::auth::Auth::new(),
         ctlr_tx,
         session: None,
@@ -778,6 +765,13 @@ async fn run_controller(
         tracing::info!("No token / actor_name on disk, starting in signed-out state");
     }
 
+    if !ran_before::get().await? {
+        let win = app
+            .get_window("welcome")
+            .context("Couldn't get handle to Welcome window")?;
+        win.show()?;
+    }
+
     let mut have_internet =
         network_changes::check_internet().context("Failed initial check for internet")?;
     tracing::info!(?have_internet);
diff --git a/rust/gui-client/src-tauri/src/client/gui/ran_before.rs b/rust/gui-client/src-tauri/src/client/gui/ran_before.rs
new file mode 100644
index 000000000..6de871ca7
--- /dev/null
+++ b/rust/gui-client/src-tauri/src/client/gui/ran_before.rs
@@ -0,0 +1,29 @@
+//! Controls an on-disk flag indicating whether the user has signed in before
+
+use anyhow::{Context as _, Result};
+use std::path::PathBuf;
+use tokio::fs;
+
+/// Returns true if the flag is set
+pub(crate) async fn get() -> Result<bool> {
+    Ok(fs::try_exists(path()?).await?)
+}
+
+/// Sets the flag to true
+pub(crate) async fn set() -> Result<()> {
+    let path = path()?;
+    fs::create_dir_all(
+        path.parent()
+            .context("ran_before path should have a parent dir")?,
+    )
+    .await?;
+    fs::write(&path, &[]).await?;
+    debug_assert!(get().await?);
+    Ok(())
+}
+
+fn path() -> Result<PathBuf> {
+    let session_dir =
+        firezone_headless_client::known_dirs::session().context("Couldn't find session dir")?;
+    Ok(session_dir.join("ran_before.txt"))
+}
diff --git a/scripts/tests/smoke-test-gui-linux.sh b/scripts/tests/smoke-test-gui-linux.sh
index 0bce34de3..dfb2fc5f3 100755
--- a/scripts/tests/smoke-test-gui-linux.sh
+++ b/scripts/tests/smoke-test-gui-linux.sh
@@ -58,7 +58,8 @@ function smoke_test() {
     bash -c "stat \"${LOGS_PATH}/\"connlib*log"
     stat "$SETTINGS_PATH"
     # stat "$DEVICE_ID_PATH"
-    stat "$RAN_BEFORE_PATH"
+    # `ran_before` is now only written after a successful sign-in
+    stat "$RAN_BEFORE_PATH" && exit 1
 
     # Run the test again and make sure the device ID is not changed
     run_fz_gui --no-deep-links smoke-test
@@ -74,7 +75,7 @@ function smoke_test() {
     rm -rf "$LOGS_PATH"
     rm "$SETTINGS_PATH"
     # rm "$DEVICE_ID_PATH"
-    rm "$RAN_BEFORE_PATH"
+    rm -f "$RAN_BEFORE_PATH"
 }
 
 function crash_test() {