diff --git a/.github/workflows/_rust.yml b/.github/workflows/_rust.yml
index f22851c8d..6119962bd 100644
--- a/.github/workflows/_rust.yml
+++ b/.github/workflows/_rust.yml
@@ -108,10 +108,12 @@ jobs:
         run: pnpm tailwindcss -i src/input.css -o src/output.css
       - name: Build client
         run: cargo build -p firezone-gui-client
-      - name: Run smoke test
-        run: cargo run -p firezone-gui-client -- smoke-test
-      - name: Test that the crash handler produces a crash dump
-        run: bash scripts/crash-handling-smoke-test.bash
+      - name: Run smoke tests (Linux)
+        if: ${{ runner.os == 'Linux' }}
+        run: bash ../../scripts/tests/smoke-test-gui-linux.sh
+      - name: Run smoke tests (Windows)
+        if: ${{ runner.os == 'Windows' }}
+        run: bash ../../scripts/tests/smoke-test-gui-windows.sh
 
   # This should be identical to `build-push-windows-release-artifacts` in `cd.yml` except for the Github permissions, needs tag, and uploading step
   build-gui:
diff --git a/scripts/tests/smoke-test-gui-linux.sh b/scripts/tests/smoke-test-gui-linux.sh
new file mode 100755
index 000000000..c94d90b26
--- /dev/null
+++ b/scripts/tests/smoke-test-gui-linux.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+BUNDLE_ID="dev.firezone.client"
+DUMP_PATH="$BUNDLE_ID/data/logs/last_crash.dmp"
+export FIREZONE_DISABLE_SYSTRAY=true
+PACKAGE=firezone-gui-client
+export RUST_LOG=firezone_gui_client=debug,warn
+export WEBKIT_DISABLE_COMPOSITING_MODE=1
+
+# Run the smoke test normally
+xvfb-run --auto-servernum cargo run -p "$PACKAGE" -- smoke-test
+
+# Delete the crash file if present
+rm -f "$DUMP_PATH"
+
+# Fail if it returns success, this is supposed to crash
+xvfb-run --auto-servernum cargo run -p "$PACKAGE" -- --crash && exit 1
+
+# Fail if the crash file wasn't written
+stat "$DUMP_PATH"
+rm "$DUMP_PATH"
+
+# I'm not sure if the last command is handled specially, so explicitly exit with 0
+exit 0
diff --git a/scripts/tests/smoke-test-gui-windows.sh b/scripts/tests/smoke-test-gui-windows.sh
new file mode 100755
index 000000000..b2466621c
--- /dev/null
+++ b/scripts/tests/smoke-test-gui-windows.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# This script must run from an elevated shell so that Firezone won't try to elevate
+
+set -euo pipefail
+
+BUNDLE_ID="dev.firezone.client"
+DUMP_PATH="$LOCALAPPDATA/$BUNDLE_ID/data/logs/last_crash.dmp"
+PACKAGE=firezone-gui-client
+
+# Run the smoke test normally
+cargo run -p "$PACKAGE" -- smoke-test
+
+# Delete the crash file if present
+rm -f "$DUMP_PATH"
+
+# Fail if it returns success, this is supposed to crash
+cargo run -p "$PACKAGE" -- --crash && exit 1
+
+# Fail if the crash file wasn't written
+stat "$DUMP_PATH"
+rm "$DUMP_PATH"
+
+# I'm not sure if the last command is handled specially, so explicitly exit with 0
+exit 0